[ 500.733878] env[63273]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'linux_bridge' {{(pid=63273) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 500.734222] env[63273]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'noop' {{(pid=63273) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 500.734259] env[63273]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'ovs' {{(pid=63273) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 500.734598] env[63273]: INFO os_vif [-] Loaded VIF plugins: linux_bridge, noop, ovs [ 500.832984] env[63273]: DEBUG oslo_concurrency.processutils [-] Running cmd (subprocess): grep -F node.session.scan /sbin/iscsiadm {{(pid=63273) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:390}} [ 500.844067] env[63273]: DEBUG oslo_concurrency.processutils [-] CMD "grep -F node.session.scan /sbin/iscsiadm" returned: 0 in 0.011s {{(pid=63273) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:428}} [ 500.997289] env[63273]: INFO nova.virt.driver [None req-53e32f29-3827-4db0-9129-91367fc49de7 None None] Loading compute driver 'vmwareapi.VMwareVCDriver' [ 501.072951] env[63273]: DEBUG oslo_concurrency.lockutils [-] Acquiring lock "oslo_vmware_api_lock" by "oslo_vmware.api.VMwareAPISession._create_session" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 501.073213] env[63273]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" acquired by "oslo_vmware.api.VMwareAPISession._create_session" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 501.073255] env[63273]: DEBUG oslo_vmware.service [-] Creating suds client with soap_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk' and wsdl_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk/vimService.wsdl' {{(pid=63273) __init__ /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:242}} [ 504.181844] env[63273]: DEBUG oslo_vmware.service [-] Invoking ServiceInstance.RetrieveServiceContent with opID=oslo.vmware-1824eb15-01b8-4e3e-bce4-8965da0fb82a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.197889] env[63273]: DEBUG oslo_vmware.api [-] Logging into host: vc1.osci.c.eu-de-1.cloud.sap. {{(pid=63273) _create_session /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:242}} [ 504.198069] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.Login with opID=oslo.vmware-aec3cbf1-d0c0-414f-9078-1f65f67501fa {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.232306] env[63273]: INFO oslo_vmware.api [-] Successfully established new session; session ID is 5c288. [ 504.232482] env[63273]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" "released" by "oslo_vmware.api.VMwareAPISession._create_session" :: held 3.159s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 504.233370] env[63273]: INFO nova.virt.vmwareapi.driver [None req-53e32f29-3827-4db0-9129-91367fc49de7 None None] VMware vCenter version: 7.0.3 [ 504.236938] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fdd22cb0-8151-4a9b-b22e-de1b0acf90ed {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.259303] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09e166cd-8548-4bd1-bf9f-b7f8760a875e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.265911] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-894d655e-352a-42a9-911f-43d20a42171f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.273566] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d16fcf2d-9cae-4610-932b-20dcb18d5de6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.287152] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa53fe4f-253e-49ec-be4d-873a83bd2d33 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.293910] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65d29db4-cd29-40cd-baef-63a69edcb69b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.326022] env[63273]: DEBUG oslo_vmware.service [-] Invoking ExtensionManager.FindExtension with opID=oslo.vmware-618b214b-5a1c-45e2-a4ed-2a281634b937 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.332336] env[63273]: DEBUG nova.virt.vmwareapi.driver [None req-53e32f29-3827-4db0-9129-91367fc49de7 None None] Extension org.openstack.compute already exists. {{(pid=63273) _register_openstack_extension /opt/stack/nova/nova/virt/vmwareapi/driver.py:224}} [ 504.335096] env[63273]: INFO nova.compute.provider_config [None req-53e32f29-3827-4db0-9129-91367fc49de7 None None] No provider configs found in /etc/nova/provider_config/. If files are present, ensure the Nova process has access. [ 504.355024] env[63273]: DEBUG nova.context [None req-53e32f29-3827-4db0-9129-91367fc49de7 None None] Found 2 cells: 00000000-0000-0000-0000-000000000000(cell0),8245bbee-0fb3-465f-9b02-27f369259d0f(cell1) {{(pid=63273) load_cells /opt/stack/nova/nova/context.py:464}} [ 504.357054] env[63273]: DEBUG oslo_concurrency.lockutils [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 504.357286] env[63273]: DEBUG oslo_concurrency.lockutils [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 504.357979] env[63273]: DEBUG oslo_concurrency.lockutils [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 504.358418] env[63273]: DEBUG oslo_concurrency.lockutils [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Acquiring lock "8245bbee-0fb3-465f-9b02-27f369259d0f" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 504.358617] env[63273]: DEBUG oslo_concurrency.lockutils [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Lock "8245bbee-0fb3-465f-9b02-27f369259d0f" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 504.359699] env[63273]: DEBUG oslo_concurrency.lockutils [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Lock "8245bbee-0fb3-465f-9b02-27f369259d0f" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 504.380535] env[63273]: INFO dbcounter [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Registered counter for database nova_cell0 [ 504.388689] env[63273]: INFO dbcounter [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Registered counter for database nova_cell1 [ 504.391914] env[63273]: DEBUG oslo_db.sqlalchemy.engines [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=63273) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 504.392297] env[63273]: DEBUG oslo_db.sqlalchemy.engines [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=63273) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 504.396941] env[63273]: DEBUG dbcounter [-] [63273] Writer thread running {{(pid=63273) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 504.397737] env[63273]: DEBUG dbcounter [-] [63273] Writer thread running {{(pid=63273) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 504.400192] env[63273]: ERROR nova.db.main.api [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main [ 504.400192] env[63273]: result = function(*args, **kwargs) [ 504.400192] env[63273]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 504.400192] env[63273]: return func(*args, **kwargs) [ 504.400192] env[63273]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 504.400192] env[63273]: result = fn(*args, **kwargs) [ 504.400192] env[63273]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 504.400192] env[63273]: return f(*args, **kwargs) [ 504.400192] env[63273]: File "/opt/stack/nova/nova/objects/service.py", line 553, in _db_service_get_minimum_version [ 504.400192] env[63273]: return db.service_get_minimum_version(context, binaries) [ 504.400192] env[63273]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 504.400192] env[63273]: _check_db_access() [ 504.400192] env[63273]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 504.400192] env[63273]: stacktrace = ''.join(traceback.format_stack()) [ 504.400192] env[63273]: [ 504.400994] env[63273]: ERROR nova.db.main.api [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main [ 504.400994] env[63273]: result = function(*args, **kwargs) [ 504.400994] env[63273]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 504.400994] env[63273]: return func(*args, **kwargs) [ 504.400994] env[63273]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 504.400994] env[63273]: result = fn(*args, **kwargs) [ 504.400994] env[63273]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 504.400994] env[63273]: return f(*args, **kwargs) [ 504.400994] env[63273]: File "/opt/stack/nova/nova/objects/service.py", line 553, in _db_service_get_minimum_version [ 504.400994] env[63273]: return db.service_get_minimum_version(context, binaries) [ 504.400994] env[63273]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 504.400994] env[63273]: _check_db_access() [ 504.400994] env[63273]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 504.400994] env[63273]: stacktrace = ''.join(traceback.format_stack()) [ 504.400994] env[63273]: [ 504.401421] env[63273]: WARNING nova.objects.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Failed to get minimum service version for cell 00000000-0000-0000-0000-000000000000 [ 504.401518] env[63273]: WARNING nova.objects.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Failed to get minimum service version for cell 8245bbee-0fb3-465f-9b02-27f369259d0f [ 504.401949] env[63273]: DEBUG oslo_concurrency.lockutils [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Acquiring lock "singleton_lock" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 504.402123] env[63273]: DEBUG oslo_concurrency.lockutils [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Acquired lock "singleton_lock" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 504.402371] env[63273]: DEBUG oslo_concurrency.lockutils [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Releasing lock "singleton_lock" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 504.402692] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Full set of CONF: {{(pid=63273) _wait_for_exit_or_signal /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:363}} [ 504.402836] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ******************************************************************************** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2600}} [ 504.402966] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] Configuration options gathered from: {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2601}} [ 504.403120] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] command line args: ['--config-file', '/etc/nova/nova.conf', '--config-file', '/etc/nova/nova-cpu-common.conf', '--config-file', '/etc/nova/nova-cpu-1.conf'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2602}} [ 504.403311] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] config files: ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2603}} [ 504.403440] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ================================================================================ {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2605}} [ 504.403657] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] allow_resize_to_same_host = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.403834] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] arq_binding_timeout = 300 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.403972] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] backdoor_port = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.404119] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] backdoor_socket = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.404290] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] block_device_allocate_retries = 60 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.404454] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] block_device_allocate_retries_interval = 3 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.404628] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cert = self.pem {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.404802] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute_driver = vmwareapi.VMwareVCDriver {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.404974] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute_monitors = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.405158] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] config_dir = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.405334] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] config_drive_format = iso9660 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.405470] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] config_file = ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.405674] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] config_source = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.405912] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] console_host = devstack {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.406112] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] control_exchange = nova {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.406278] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cpu_allocation_ratio = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.406441] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] daemon = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.406610] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] debug = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.406775] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] default_access_ip_network_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.406950] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] default_availability_zone = nova {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.407132] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] default_ephemeral_format = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.407301] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] default_green_pool_size = 1000 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.407553] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'glanceclient=WARN', 'oslo.privsep.daemon=INFO'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.407734] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] default_schedule_zone = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.407900] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] disk_allocation_ratio = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.408078] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] enable_new_services = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.408266] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] enabled_apis = ['osapi_compute'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.408488] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] enabled_ssl_apis = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.408592] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] flat_injected = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.408774] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] force_config_drive = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.408914] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] force_raw_images = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.409097] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] graceful_shutdown_timeout = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.409266] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] heal_instance_info_cache_interval = 60 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.409484] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] host = cpu-1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.409679] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] initial_cpu_allocation_ratio = 4.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.409865] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] initial_disk_allocation_ratio = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.410042] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] initial_ram_allocation_ratio = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.410269] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] injected_network_template = /opt/stack/nova/nova/virt/interfaces.template {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.410440] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] instance_build_timeout = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.410606] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] instance_delete_interval = 300 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.410816] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] instance_format = [instance: %(uuid)s] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.410989] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] instance_name_template = instance-%08x {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.411173] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] instance_usage_audit = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.411352] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] instance_usage_audit_period = month {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.411524] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] instance_uuid_format = [instance: %(uuid)s] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.411693] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] instances_path = /opt/stack/data/nova/instances {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.411863] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] internal_service_availability_zone = internal {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.412035] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] key = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.412202] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] live_migration_retry_count = 30 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.412369] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] log_config_append = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.412542] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.412705] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] log_dir = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.412868] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] log_file = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.412999] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] log_options = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.413178] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] log_rotate_interval = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.413352] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] log_rotate_interval_type = days {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.413521] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] log_rotation_type = none {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.413654] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.413785] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.413964] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.414149] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.414281] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.414445] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] long_rpc_timeout = 1800 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.414607] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] max_concurrent_builds = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.414766] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] max_concurrent_live_migrations = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.414926] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] max_concurrent_snapshots = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.415097] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] max_local_block_devices = 3 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.415259] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] max_logfile_count = 30 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.415416] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] max_logfile_size_mb = 200 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.415574] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] maximum_instance_delete_attempts = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.415746] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] metadata_listen = 0.0.0.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.415916] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] metadata_listen_port = 8775 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.416096] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] metadata_workers = 2 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.416263] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] migrate_max_retries = -1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.416432] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] mkisofs_cmd = genisoimage {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.416642] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] my_block_storage_ip = 10.180.1.21 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.416779] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] my_ip = 10.180.1.21 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.416945] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] network_allocate_retries = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.417139] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] non_inheritable_image_properties = ['cache_in_nova', 'bittorrent'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.417316] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] osapi_compute_listen = 0.0.0.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.417481] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] osapi_compute_listen_port = 8774 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.417682] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] osapi_compute_unique_server_name_scope = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.417862] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] osapi_compute_workers = 2 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.418038] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] password_length = 12 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.418221] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] periodic_enable = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.418375] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] periodic_fuzzy_delay = 60 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.418594] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] pointer_model = usbtablet {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.418716] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] preallocate_images = none {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.418892] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] publish_errors = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.419020] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] pybasedir = /opt/stack/nova {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.419182] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ram_allocation_ratio = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.419346] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] rate_limit_burst = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.419514] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] rate_limit_except_level = CRITICAL {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.419699] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] rate_limit_interval = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.419874] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] reboot_timeout = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.420049] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] reclaim_instance_interval = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.420217] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] record = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.420389] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] reimage_timeout_per_gb = 60 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.420558] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] report_interval = 120 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.420742] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] rescue_timeout = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.420914] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] reserved_host_cpus = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.421088] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] reserved_host_disk_mb = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.421255] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] reserved_host_memory_mb = 512 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.421417] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] reserved_huge_pages = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.421578] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] resize_confirm_window = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.421739] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] resize_fs_using_block_device = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.421918] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] resume_guests_state_on_host_boot = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.422097] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] rootwrap_config = /etc/nova/rootwrap.conf {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.422264] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] rpc_response_timeout = 60 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.422427] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] run_external_periodic_tasks = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.422600] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] running_deleted_instance_action = reap {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.422769] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] running_deleted_instance_poll_interval = 1800 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.422934] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] running_deleted_instance_timeout = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.423107] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] scheduler_instance_sync_interval = 120 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.423281] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] service_down_time = 720 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.423455] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] servicegroup_driver = db {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.423619] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] shelved_offload_time = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.423781] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] shelved_poll_interval = 3600 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.423952] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] shutdown_timeout = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.424128] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] source_is_ipv6 = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.424291] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ssl_only = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.424545] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] state_path = /opt/stack/data/n-cpu-1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.424716] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] sync_power_state_interval = 600 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.424881] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] sync_power_state_pool_size = 1000 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.425137] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] syslog_log_facility = LOG_USER {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.425435] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] tempdir = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.425637] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] timeout_nbd = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.425823] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] transport_url = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.425994] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] update_resources_interval = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.426176] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] use_cow_images = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.426340] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] use_eventlog = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.426501] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] use_journal = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.426674] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] use_json = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.427038] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] use_rootwrap_daemon = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.427195] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] use_stderr = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.427369] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] use_syslog = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.427534] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vcpu_pin_set = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.427762] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plugging_is_fatal = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.427878] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plugging_timeout = 300 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.428060] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] virt_mkfs = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.428228] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] volume_usage_poll_interval = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.428389] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] watch_log_file = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.428569] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] web = /usr/share/spice-html5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 504.428760] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_concurrency.disable_process_locking = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.429088] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_concurrency.lock_path = /opt/stack/data/n-cpu-1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.429294] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.429467] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_metrics.metrics_enabled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.429671] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_metrics.metrics_process_name = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.429892] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.430091] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.430286] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.auth_strategy = keystone {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.430457] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.compute_link_prefix = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.430652] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.430815] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.dhcp_domain = novalocal {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.430986] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.enable_instance_password = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.431165] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.glance_link_prefix = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.431332] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.instance_list_cells_batch_fixed_size = 100 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.431507] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.instance_list_cells_batch_strategy = distributed {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.431674] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.instance_list_per_project_cells = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.431838] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.list_records_by_skipping_down_cells = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.432040] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.local_metadata_per_cell = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.432183] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.max_limit = 1000 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.432354] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.metadata_cache_expiration = 15 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.432533] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.neutron_default_tenant_id = default {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.432704] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.use_neutron_default_nets = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.432888] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.vendordata_dynamic_connect_timeout = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.433067] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.vendordata_dynamic_failure_fatal = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.433267] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.vendordata_dynamic_read_timeout = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.433423] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.vendordata_dynamic_ssl_certfile = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.433597] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.vendordata_dynamic_targets = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.433831] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.vendordata_jsonfile_path = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.433950] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api.vendordata_providers = ['StaticJSON'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.434165] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.backend = dogpile.cache.memcached {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.434341] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.backend_argument = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.434516] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.config_prefix = cache.oslo {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.434691] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.dead_timeout = 60.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.434861] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.debug_cache_backend = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.435036] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.enable_retry_client = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.435206] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.enable_socket_keepalive = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.435380] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.enabled = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.435546] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.enforce_fips_mode = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.435714] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.expiration_time = 600 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.435881] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.hashclient_retry_attempts = 2 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.436059] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.hashclient_retry_delay = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.436233] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.memcache_dead_retry = 300 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.436395] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.memcache_password = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.436561] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.memcache_pool_connection_get_timeout = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.436726] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.memcache_pool_flush_on_reconnect = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.436893] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.memcache_pool_maxsize = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.437066] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.memcache_pool_unused_timeout = 60 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.437234] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.memcache_sasl_enabled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.437418] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.memcache_servers = ['localhost:11211'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.437612] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.memcache_socket_timeout = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.437781] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.memcache_username = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.437953] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.proxies = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.438131] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.redis_password = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.438308] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.redis_sentinel_service_name = mymaster {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.438485] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.redis_sentinels = ['localhost:26379'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.438660] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.redis_server = localhost:6379 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.438829] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.redis_socket_timeout = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.438990] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.redis_username = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.439169] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.retry_attempts = 2 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.439338] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.retry_delay = 0.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.439504] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.socket_keepalive_count = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.439693] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.socket_keepalive_idle = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.439869] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.socket_keepalive_interval = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.440046] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.tls_allowed_ciphers = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.440212] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.tls_cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.440372] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.tls_certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.440537] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.tls_enabled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.440762] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cache.tls_keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.440924] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.auth_section = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.441117] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.auth_type = password {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.441287] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.441468] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.catalog_info = volumev3::publicURL {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.441632] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.441799] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.441964] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.cross_az_attach = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.442146] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.debug = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.442312] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.endpoint_template = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.442480] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.http_retries = 3 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.442647] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.442810] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.442984] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.os_region_name = RegionOne {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.443166] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.443330] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cinder.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.443507] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.consecutive_build_service_disable_threshold = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.443670] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.cpu_dedicated_set = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.443831] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.cpu_shared_set = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.443999] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.image_type_exclude_list = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.444180] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.live_migration_wait_for_vif_plug = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.444348] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.max_concurrent_disk_ops = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.444516] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.max_disk_devices_to_attach = -1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.444682] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.packing_host_numa_cells_allocation_strategy = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.444855] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.provider_config_location = /etc/nova/provider_config/ {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.445033] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.resource_provider_association_refresh = 300 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.445195] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.sharing_providers_max_uuids_per_request = 200 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.445361] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.shutdown_retry_interval = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.445544] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] compute.vmdk_allowed_types = ['streamOptimized', 'monolithicSparse'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.445726] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] conductor.workers = 2 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.445905] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] console.allowed_origins = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.446077] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] console.ssl_ciphers = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.446255] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] console.ssl_minimum_version = default {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.446425] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] consoleauth.enforce_session_timeout = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.446595] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] consoleauth.token_ttl = 600 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.446764] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.446925] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.447102] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.447267] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.connect_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.447427] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.connect_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.447602] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.endpoint_override = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.447768] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.447932] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.448108] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.max_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.448270] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.min_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.448430] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.region_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.448589] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.retriable_status_codes = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.448749] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.service_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.448921] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.service_type = accelerator {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.449097] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.449258] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.status_code_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.449416] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.status_code_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.449579] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.449798] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.valid_interfaces = ['internal', 'public'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.449965] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] cyborg.version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.450164] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.backend = sqlalchemy {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.450340] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.connection = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.450508] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.connection_debug = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.450717] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.connection_parameters = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.450896] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.connection_recycle_time = 3600 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.451067] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.connection_trace = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.451237] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.db_inc_retry_interval = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.451438] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.db_max_retries = 20 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.451636] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.db_max_retry_interval = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.451828] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.db_retry_interval = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.452027] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.max_overflow = 50 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.453123] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.max_pool_size = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.453123] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.max_retries = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.453123] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.mysql_sql_mode = TRADITIONAL {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.453123] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.mysql_wsrep_sync_wait = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.453123] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.pool_timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.453337] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.retry_interval = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.453451] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.slave_connection = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.453689] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.sqlite_synchronous = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.453894] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] database.use_db_reconnect = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.454107] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.backend = sqlalchemy {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.454290] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.connection = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.454464] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.connection_debug = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.454639] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.connection_parameters = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.454808] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.connection_recycle_time = 3600 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.454973] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.connection_trace = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.455150] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.db_inc_retry_interval = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.455316] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.db_max_retries = 20 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.455481] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.db_max_retry_interval = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.455644] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.db_retry_interval = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.455808] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.max_overflow = 50 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.455972] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.max_pool_size = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.456148] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.max_retries = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.456321] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.mysql_sql_mode = TRADITIONAL {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.456479] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.mysql_wsrep_sync_wait = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.456639] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.pool_timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.456833] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.retry_interval = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.456985] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.slave_connection = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.457223] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] api_database.sqlite_synchronous = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.457414] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] devices.enabled_mdev_types = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.457599] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ephemeral_storage_encryption.cipher = aes-xts-plain64 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.457789] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ephemeral_storage_encryption.default_format = luks {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.457960] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ephemeral_storage_encryption.enabled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.458145] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ephemeral_storage_encryption.key_size = 512 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.458312] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.api_servers = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.458478] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.458643] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.458811] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.459026] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.connect_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.459145] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.connect_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.459381] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.debug = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.459477] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.default_trusted_certificate_ids = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.459653] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.enable_certificate_validation = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.459835] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.enable_rbd_download = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.460018] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.endpoint_override = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.460185] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.460348] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.460513] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.max_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.460702] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.min_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.460881] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.num_retries = 3 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.461068] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.rbd_ceph_conf = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.461238] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.rbd_connect_timeout = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.461411] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.rbd_pool = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.461579] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.rbd_user = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.461743] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.region_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.461903] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.retriable_status_codes = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.462078] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.service_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.462254] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.service_type = image {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.462855] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.462855] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.status_code_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.462993] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.status_code_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.463239] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.463514] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.valid_interfaces = ['internal', 'public'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.463770] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.verify_glance_signatures = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.464032] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] glance.version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.464305] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] guestfs.debug = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.464557] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] mks.enabled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.465067] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] mks.mksproxy_base_url = http://127.0.0.1:6090/ {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.465359] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] image_cache.manager_interval = 2400 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.465655] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] image_cache.precache_concurrency = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.465909] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] image_cache.remove_unused_base_images = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.466172] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] image_cache.remove_unused_original_minimum_age_seconds = 86400 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.466413] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] image_cache.remove_unused_resized_minimum_age_seconds = 3600 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.466674] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] image_cache.subdirectory_name = _base {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.466939] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.api_max_retries = 60 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.467203] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.api_retry_interval = 2 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.467450] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.auth_section = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.467714] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.auth_type = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.467966] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.468259] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.468485] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.468753] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.conductor_group = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.469085] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.connect_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.469293] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.connect_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.469566] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.endpoint_override = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.469852] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.470129] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.470394] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.max_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.470650] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.min_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.470917] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.peer_list = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.471150] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.region_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.471339] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.retriable_status_codes = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.471516] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.serial_console_state_timeout = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.471683] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.service_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.471888] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.service_type = baremetal {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.472096] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.shard = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.472275] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.472439] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.status_code_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.472602] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.status_code_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.472766] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.472954] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.valid_interfaces = ['internal', 'public'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.473133] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ironic.version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.473325] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] key_manager.backend = nova.keymgr.conf_key_mgr.ConfKeyManager {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.473531] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] key_manager.fixed_key = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.473773] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.auth_endpoint = http://localhost/identity/v3 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.473991] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.barbican_api_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.474189] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.barbican_endpoint = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.474370] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.barbican_endpoint_type = public {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.474533] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.barbican_region_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.474697] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.474857] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.475028] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.475198] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.475357] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.475521] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.number_of_retries = 60 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.475683] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.retry_delay = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.475851] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.send_service_user_token = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.476021] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.476187] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.476389] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.verify_ssl = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.476564] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican.verify_ssl_path = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.476735] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican_service_user.auth_section = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.476904] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican_service_user.auth_type = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.477078] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican_service_user.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.477240] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican_service_user.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.477406] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican_service_user.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.477570] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican_service_user.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.477730] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican_service_user.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.477892] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican_service_user.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.478141] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] barbican_service_user.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.478375] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.approle_role_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.478487] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.approle_secret_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.478649] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.478808] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.478972] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.479200] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.479298] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.479473] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.kv_mountpoint = secret {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.479654] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.kv_path = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.479830] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.kv_version = 2 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.479998] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.namespace = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.480175] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.root_token_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.480341] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.480499] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.ssl_ca_crt_file = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.480680] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.480859] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.use_ssl = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.481044] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vault.vault_url = http://127.0.0.1:8200 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.481222] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.auth_section = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.481390] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.auth_type = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.481590] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.481763] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.481932] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.482110] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.connect_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.482272] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.connect_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.482430] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.endpoint_override = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.482593] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.482751] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.482943] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.max_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.483164] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.min_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.483338] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.region_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.483496] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.retriable_status_codes = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.483660] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.service_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.483833] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.service_type = identity {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.483998] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.484175] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.status_code_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.484341] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.status_code_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.484503] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.484689] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.valid_interfaces = ['internal', 'public'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.484854] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] keystone.version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.485071] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.connection_uri = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.485238] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.cpu_mode = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.485407] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.cpu_model_extra_flags = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.485579] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.cpu_models = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.485753] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.cpu_power_governor_high = performance {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.485926] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.cpu_power_governor_low = powersave {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.486101] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.cpu_power_management = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.486280] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.cpu_power_management_strategy = cpu_state {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.486447] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.device_detach_attempts = 8 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.486609] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.device_detach_timeout = 20 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.486776] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.disk_cachemodes = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.486936] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.disk_prefix = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.487120] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.enabled_perf_events = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.487287] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.file_backed_memory = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.487451] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.gid_maps = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.487611] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.hw_disk_discard = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.487778] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.hw_machine_type = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.487958] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.images_rbd_ceph_conf = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.488143] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.images_rbd_glance_copy_poll_interval = 15 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.488312] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.images_rbd_glance_copy_timeout = 600 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.488497] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.images_rbd_glance_store_name = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.488650] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.images_rbd_pool = rbd {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.488823] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.images_type = default {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.488982] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.images_volume_group = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.489158] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.inject_key = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.489320] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.inject_partition = -2 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.489511] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.inject_password = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.489793] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.iscsi_iface = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.489890] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.iser_use_multipath = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.490057] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_bandwidth = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.490232] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_completion_timeout = 800 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.490395] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_downtime = 500 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.490560] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_downtime_delay = 75 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.490836] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_downtime_steps = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.490905] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_inbound_addr = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.491047] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_permit_auto_converge = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.491217] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_permit_post_copy = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.491382] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_scheme = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.491559] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_timeout_action = abort {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.491736] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_tunnelled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.491893] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_uri = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.492069] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.live_migration_with_native_tls = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.492237] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.max_queues = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.492406] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.mem_stats_period_seconds = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.492655] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.migration_inbound_addr = 10.180.1.21 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.492822] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.nfs_mount_options = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.493141] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.nfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.493316] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.num_aoe_discover_tries = 3 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.493482] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.num_iser_scan_tries = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.493646] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.num_memory_encrypted_guests = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.493815] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.num_nvme_discover_tries = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.493979] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.num_pcie_ports = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.494194] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.num_volume_scan_tries = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.494382] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.pmem_namespaces = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.494546] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.quobyte_client_cfg = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.494847] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.quobyte_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.495034] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.rbd_connect_timeout = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.495209] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.rbd_destroy_volume_retries = 12 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.495379] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.rbd_destroy_volume_retry_interval = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.495543] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.rbd_secret_uuid = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.495701] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.rbd_user = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.495868] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.realtime_scheduler_priority = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.496050] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.remote_filesystem_transport = ssh {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.496215] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.rescue_image_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.496374] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.rescue_kernel_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.496532] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.rescue_ramdisk_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.496701] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.rng_dev_path = /dev/urandom {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.496860] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.rx_queue_size = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.497042] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.smbfs_mount_options = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.497326] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.smbfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.497555] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.snapshot_compression = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.497739] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.snapshot_image_format = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.497966] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.snapshots_directory = /opt/stack/data/nova/instances/snapshots {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.498150] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.sparse_logical_volumes = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.498319] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.swtpm_enabled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.498492] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.swtpm_group = tss {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.498666] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.swtpm_user = tss {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.498844] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.sysinfo_serial = unique {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.499012] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.tb_cache_size = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.499177] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.tx_queue_size = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.499342] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.uid_maps = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.499506] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.use_virtio_for_bridges = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.499712] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.virt_type = kvm {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.499927] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.volume_clear = zero {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.500085] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.volume_clear_size = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.500248] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.volume_use_multipath = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.500411] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.vzstorage_cache_path = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.500583] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.vzstorage_log_path = /var/log/vstorage/%(cluster_name)s/nova.log.gz {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.500755] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.vzstorage_mount_group = qemu {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.500959] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.vzstorage_mount_opts = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.501158] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.vzstorage_mount_perms = 0770 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.501449] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.vzstorage_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.501643] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.vzstorage_mount_user = stack {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.501822] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] libvirt.wait_soft_reboot_seconds = 120 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.502191] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.auth_section = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.502260] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.auth_type = password {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.502381] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.502541] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.502707] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.502871] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.connect_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.503041] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.connect_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.503218] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.default_floating_pool = public {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.503378] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.endpoint_override = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.503541] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.extension_sync_interval = 600 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.503707] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.http_retries = 3 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.503871] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.504042] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.504272] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.max_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.504467] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.metadata_proxy_shared_secret = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.504636] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.min_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.504813] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.ovs_bridge = br-int {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.504978] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.physnets = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.505166] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.region_name = RegionOne {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.505332] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.retriable_status_codes = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.505506] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.service_metadata_proxy = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.505667] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.service_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.505839] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.service_type = network {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.506010] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.506184] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.status_code_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.506348] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.status_code_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.506506] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.506688] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.valid_interfaces = ['internal', 'public'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.506851] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] neutron.version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.507035] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] notifications.bdms_in_notifications = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.507220] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] notifications.default_level = INFO {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.507399] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] notifications.notification_format = unversioned {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.507571] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] notifications.notify_on_state_change = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.507749] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] notifications.versioned_notifications_topics = ['versioned_notifications'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.507928] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] pci.alias = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.508112] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] pci.device_spec = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.508283] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] pci.report_in_placement = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.508454] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.auth_section = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.508627] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.auth_type = password {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.508797] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.auth_url = http://10.180.1.21/identity {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.508959] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.509128] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.509295] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.509545] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.connect_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.509682] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.connect_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.509864] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.default_domain_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.510070] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.default_domain_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.510206] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.domain_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.510365] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.domain_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.510524] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.endpoint_override = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.510691] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.510849] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.511013] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.max_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.511175] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.min_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.511345] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.password = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.511505] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.project_domain_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.511673] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.project_domain_name = Default {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.511840] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.project_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.512049] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.project_name = service {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.512245] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.region_name = RegionOne {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.512413] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.retriable_status_codes = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.512576] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.service_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.512746] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.service_type = placement {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.512911] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.513086] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.status_code_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.513253] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.status_code_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.513414] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.system_scope = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.513573] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.513736] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.trust_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.513898] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.user_domain_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.514079] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.user_domain_name = Default {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.514243] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.user_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.514416] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.username = placement {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.514602] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.valid_interfaces = ['internal', 'public'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.514766] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] placement.version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.514952] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.cores = 20 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.515132] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.count_usage_from_placement = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.515309] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.driver = nova.quota.DbQuotaDriver {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.515487] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.injected_file_content_bytes = 10240 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.515658] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.injected_file_path_length = 255 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.515827] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.injected_files = 5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.515997] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.instances = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.516178] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.key_pairs = 100 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.516346] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.metadata_items = 128 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.516514] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.ram = 51200 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.516679] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.recheck_quota = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.516851] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.server_group_members = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.517028] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] quota.server_groups = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.517209] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] scheduler.discover_hosts_in_cells_interval = -1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.517378] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] scheduler.enable_isolated_aggregate_filtering = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.517580] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] scheduler.image_metadata_prefilter = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.517760] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] scheduler.limit_tenants_to_placement_aggregate = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.517932] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] scheduler.max_attempts = 3 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.518109] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] scheduler.max_placement_results = 1000 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.518279] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] scheduler.placement_aggregate_required_for_tenants = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.518445] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] scheduler.query_placement_for_image_type_support = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.518610] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] scheduler.query_placement_for_routed_network_aggregates = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.518795] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] scheduler.workers = 2 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.518973] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.aggregate_image_properties_isolation_namespace = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.519159] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.aggregate_image_properties_isolation_separator = . {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.519344] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.available_filters = ['nova.scheduler.filters.all_filters'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.519516] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.build_failure_weight_multiplier = 1000000.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.519708] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.cpu_weight_multiplier = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.519890] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.cross_cell_move_weight_multiplier = 1000000.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.520069] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.disk_weight_multiplier = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.520269] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.enabled_filters = ['ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'SameHostFilter', 'DifferentHostFilter'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.520440] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.host_subset_size = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.520608] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.hypervisor_version_weight_multiplier = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.520769] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.image_properties_default_architecture = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.520935] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.io_ops_weight_multiplier = -1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.521113] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.isolated_hosts = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.521280] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.isolated_images = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.521447] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.max_instances_per_host = 50 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.521613] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.max_io_ops_per_host = 8 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.521782] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.num_instances_weight_multiplier = 0.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.521954] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.pci_in_placement = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.522161] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.pci_weight_multiplier = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.522360] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.ram_weight_multiplier = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.522530] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.restrict_isolated_hosts_to_isolated_images = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.522695] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.shuffle_best_same_weighed_hosts = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.522861] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.soft_affinity_weight_multiplier = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.523036] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.soft_anti_affinity_weight_multiplier = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.523202] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.track_instance_changes = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.523382] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] filter_scheduler.weight_classes = ['nova.scheduler.weights.all_weighers'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.523552] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] metrics.required = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.523719] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] metrics.weight_multiplier = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.523943] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] metrics.weight_of_unavailable = -10000.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.524171] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] metrics.weight_setting = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.524511] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] serial_console.base_url = ws://127.0.0.1:6083/ {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.524690] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] serial_console.enabled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.524873] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] serial_console.port_range = 10000:20000 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.525059] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] serial_console.proxyclient_address = 127.0.0.1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.525237] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] serial_console.serialproxy_host = 0.0.0.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.525407] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] serial_console.serialproxy_port = 6083 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.525577] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] service_user.auth_section = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.525752] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] service_user.auth_type = password {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.525914] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] service_user.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.526084] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] service_user.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.526252] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] service_user.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.526442] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] service_user.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.526619] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] service_user.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.526796] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] service_user.send_service_user_token = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.526961] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] service_user.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.527137] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] service_user.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.527325] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.agent_enabled = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.527490] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.enabled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.527813] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.html5proxy_base_url = http://127.0.0.1:6082/spice_auto.html {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.528023] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.html5proxy_host = 0.0.0.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.528192] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.html5proxy_port = 6082 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.528356] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.image_compression = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.528516] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.jpeg_compression = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.528675] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.playback_compression = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.528848] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.server_listen = 127.0.0.1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.529026] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.server_proxyclient_address = 127.0.0.1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.529200] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.streaming_mode = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.529373] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] spice.zlib_compression = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.529540] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] upgrade_levels.baseapi = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.529774] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] upgrade_levels.compute = auto {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.529905] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] upgrade_levels.conductor = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.530079] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] upgrade_levels.scheduler = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.530255] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vendordata_dynamic_auth.auth_section = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.530415] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vendordata_dynamic_auth.auth_type = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.530571] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vendordata_dynamic_auth.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.530733] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vendordata_dynamic_auth.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.530892] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vendordata_dynamic_auth.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.531068] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vendordata_dynamic_auth.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.531228] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vendordata_dynamic_auth.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.531387] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vendordata_dynamic_auth.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.531542] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vendordata_dynamic_auth.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.531714] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.api_retry_count = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.531876] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.ca_file = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.532071] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.cache_prefix = devstack-image-cache {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.532251] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.cluster_name = testcl1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.532419] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.connection_pool_size = 10 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.532581] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.console_delay_seconds = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.532750] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.datastore_regex = ^datastore.* {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.532966] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.host_ip = vc1.osci.c.eu-de-1.cloud.sap {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.533157] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.host_password = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.533327] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.host_port = 443 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.533516] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.host_username = administrator@vsphere.local {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.533704] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.insecure = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.533865] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.integration_bridge = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.534043] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.maximum_objects = 100 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.534209] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.pbm_default_policy = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.534372] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.pbm_enabled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.534530] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.pbm_wsdl_location = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.534698] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.serial_log_dir = /opt/vmware/vspc {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.534858] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.serial_port_proxy_uri = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.535022] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.serial_port_service_uri = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.535199] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.task_poll_interval = 0.5 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.535372] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.use_linked_clone = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.535539] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.vnc_keymap = en-us {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.535706] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.vnc_port = 5900 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.535871] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vmware.vnc_port_total = 10000 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.536069] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vnc.auth_schemes = ['none'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.536251] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vnc.enabled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.536569] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vnc.novncproxy_base_url = http://127.0.0.1:6080/vnc_auto.html {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.536757] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vnc.novncproxy_host = 0.0.0.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.536931] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vnc.novncproxy_port = 6080 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.537123] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vnc.server_listen = 127.0.0.1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.537303] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vnc.server_proxyclient_address = 127.0.0.1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.537465] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vnc.vencrypt_ca_certs = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.537657] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vnc.vencrypt_client_cert = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.537786] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vnc.vencrypt_client_key = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.537969] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.disable_compute_service_check_for_ffu = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.538152] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.disable_deep_image_inspection = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.538313] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.disable_fallback_pcpu_query = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.538475] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.disable_group_policy_check_upcall = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.538636] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.disable_libvirt_livesnapshot = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.538799] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.disable_rootwrap = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.538962] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.enable_numa_live_migration = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.539140] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.enable_qemu_monitor_announce_self = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.539302] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.ensure_libvirt_rbd_instance_dir_cleanup = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.539463] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.handle_virt_lifecycle_events = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.539639] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.libvirt_disable_apic = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.539825] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.never_download_image_if_on_rbd = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.539993] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.qemu_monitor_announce_self_count = 3 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.540173] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.qemu_monitor_announce_self_interval = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.540338] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.reserve_disk_resource_for_image_cache = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.540501] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.skip_cpu_compare_at_startup = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.540665] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.skip_cpu_compare_on_dest = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.540828] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.skip_hypervisor_version_check_on_lm = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.540992] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.skip_reserve_in_use_ironic_nodes = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544584] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.unified_limits_count_pcpu_as_vcpu = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544584] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] workarounds.wait_for_vif_plugged_event_during_hard_reboot = [] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544584] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] wsgi.api_paste_config = /etc/nova/api-paste.ini {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544584] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] wsgi.client_socket_timeout = 900 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544584] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] wsgi.default_pool_size = 1000 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544584] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] wsgi.keep_alive = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544769] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] wsgi.max_header_line = 16384 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544769] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] wsgi.secure_proxy_ssl_header = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544769] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] wsgi.ssl_ca_file = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544769] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] wsgi.ssl_cert_file = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544769] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] wsgi.ssl_key_file = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544769] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] wsgi.tcp_keepidle = 600 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544926] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] wsgi.wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544926] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] zvm.ca_file = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544926] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] zvm.cloud_connector_url = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544926] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] zvm.image_tmp_path = /opt/stack/data/n-cpu-1/images {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544926] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] zvm.reachable_timeout = 300 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.544926] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_policy.enforce_new_defaults = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.545144] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_policy.enforce_scope = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.545144] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_policy.policy_default_rule = default {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.545144] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_policy.policy_dirs = ['policy.d'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.545144] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_policy.policy_file = policy.yaml {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.545252] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_policy.remote_content_type = application/x-www-form-urlencoded {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.545573] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_policy.remote_ssl_ca_crt_file = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.545573] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_policy.remote_ssl_client_crt_file = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.545685] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_policy.remote_ssl_client_key_file = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.545846] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_policy.remote_ssl_verify_server_crt = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.546027] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.546197] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.546370] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler.connection_string = messaging:// {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.546531] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler.enabled = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.546706] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler.es_doc_type = notification {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.546870] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler.es_scroll_size = 10000 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.547044] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler.es_scroll_time = 2m {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.547206] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler.filter_error_trace = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.547371] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler.hmac_keys = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.547543] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler.sentinel_service_name = mymaster {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.547705] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler.socket_timeout = 0.1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.547870] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler.trace_requests = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.548041] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler.trace_sqlalchemy = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.548222] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler_jaeger.process_tags = {} {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.548384] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler_jaeger.service_name_prefix = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.548545] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] profiler_otlp.service_name_prefix = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.548711] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] remote_debug.host = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.548876] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] remote_debug.port = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.549063] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.549230] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.549395] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.549589] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.549783] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.549953] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.550132] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.heartbeat_in_pthread = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.550302] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.heartbeat_rate = 3 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.550461] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.heartbeat_timeout_threshold = 60 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.550629] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.hostname = devstack {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.550792] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.kombu_compression = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.550962] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.551145] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.551317] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.551488] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.processname = nova-compute {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.551656] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.551823] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.552033] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.552227] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.552396] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_quorum_delivery_limit = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.552563] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_bytes = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.552728] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_length = 0 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.552897] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.553076] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.553246] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.553411] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_stream_fanout = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.553573] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.553737] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rabbit_transient_quorum_queue = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.553907] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.554087] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.ssl = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.554266] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.ssl_ca_file = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.554440] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.ssl_cert_file = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.554605] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.554780] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.ssl_key_file = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.554950] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.ssl_version = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.555185] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_rabbit.use_queue_manager = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.555402] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_notifications.driver = ['messagingv2'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.555578] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_notifications.retry = -1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.555769] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_notifications.topics = ['notifications'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.556078] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_messaging_notifications.transport_url = **** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.556163] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.auth_section = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.556302] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.auth_type = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.556461] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.cafile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.556620] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.certfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.556783] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.collect_timing = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.556945] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.connect_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.557116] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.connect_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.557277] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.endpoint_id = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.557434] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.endpoint_override = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.557593] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.insecure = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.557753] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.keyfile = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.557911] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.max_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.558077] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.min_version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.558236] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.region_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.558391] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.retriable_status_codes = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.558547] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.service_name = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.558704] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.service_type = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.558867] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.split_loggers = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.559030] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.status_code_retries = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.559193] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.status_code_retry_delay = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.559347] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.timeout = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.559504] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.valid_interfaces = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.559690] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_limit.version = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.559872] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_reports.file_event_handler = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.560050] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_reports.file_event_handler_interval = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.560217] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] oslo_reports.log_dir = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.560392] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_linux_bridge_privileged.capabilities = [12] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.560552] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_linux_bridge_privileged.group = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.560877] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_linux_bridge_privileged.helper_command = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.560978] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_linux_bridge_privileged.logger_name = oslo_privsep.daemon {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.561078] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_linux_bridge_privileged.thread_pool_size = 8 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.561243] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_linux_bridge_privileged.user = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.561415] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_ovs_privileged.capabilities = [12, 1] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.561573] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_ovs_privileged.group = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.561728] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_ovs_privileged.helper_command = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.561896] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_ovs_privileged.logger_name = oslo_privsep.daemon {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.562081] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_ovs_privileged.thread_pool_size = 8 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.562247] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] vif_plug_ovs_privileged.user = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.562419] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_linux_bridge.flat_interface = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.562604] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_linux_bridge.forward_bridge_interface = ['all'] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.562780] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_linux_bridge.iptables_bottom_regex = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.562955] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_linux_bridge.iptables_drop_action = DROP {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.563137] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_linux_bridge.iptables_top_regex = {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.563307] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_linux_bridge.network_device_mtu = 1500 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.563477] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_linux_bridge.use_ipv6 = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.563642] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_linux_bridge.vlan_interface = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.563824] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_ovs.default_qos_type = linux-noop {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.564010] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_ovs.isolate_vif = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.564191] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_ovs.network_device_mtu = 1500 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.564366] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_ovs.ovs_vsctl_timeout = 120 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.564545] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_ovs.ovsdb_connection = tcp:127.0.0.1:6640 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.564725] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_ovs.ovsdb_interface = native {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.564891] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_vif_ovs.per_port_bridge = False {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.565070] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_brick.lock_path = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.565243] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_brick.wait_mpath_device_attempts = 4 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.565411] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] os_brick.wait_mpath_device_interval = 1 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.565578] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] privsep_osbrick.capabilities = [21] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.565738] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] privsep_osbrick.group = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.565897] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] privsep_osbrick.helper_command = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.566073] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] privsep_osbrick.logger_name = os_brick.privileged {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.566246] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] privsep_osbrick.thread_pool_size = 8 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.566405] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] privsep_osbrick.user = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.566577] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] nova_sys_admin.capabilities = [0, 1, 2, 3, 12, 21] {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.566738] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] nova_sys_admin.group = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.566900] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] nova_sys_admin.helper_command = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.567075] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] nova_sys_admin.logger_name = oslo_privsep.daemon {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.567246] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] nova_sys_admin.thread_pool_size = 8 {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.567406] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] nova_sys_admin.user = None {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 504.567546] env[63273]: DEBUG oslo_service.service [None req-64be2a23-d2b7-4886-a1fc-e817705949d0 None None] ******************************************************************************** {{(pid=63273) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2624}} [ 504.567972] env[63273]: INFO nova.service [-] Starting compute node (version 0.0.1) [ 504.578853] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Getting list of instances from cluster (obj){ [ 504.578853] env[63273]: value = "domain-c8" [ 504.578853] env[63273]: _type = "ClusterComputeResource" [ 504.578853] env[63273]: } {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 504.580179] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30c741a2-b410-41a9-8d70-515a708277d3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.589617] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Got total of 0 instances {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 504.590279] env[63273]: WARNING nova.virt.vmwareapi.driver [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] The vmwareapi driver is not tested by the OpenStack project nor does it have clear maintainer(s) and thus its quality can not be ensured. It should be considered experimental and may be removed in a future release. If you are using the driver in production please let us know via the openstack-discuss mailing list. [ 504.590756] env[63273]: INFO nova.virt.node [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Generated node identity 34a0ca39-8974-44d6-ab34-3ab6cf8432cd [ 504.590988] env[63273]: INFO nova.virt.node [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Wrote node identity 34a0ca39-8974-44d6-ab34-3ab6cf8432cd to /opt/stack/data/n-cpu-1/compute_id [ 504.603398] env[63273]: WARNING nova.compute.manager [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Compute nodes ['34a0ca39-8974-44d6-ab34-3ab6cf8432cd'] for host cpu-1 were not found in the database. If this is the first time this service is starting on this host, then you can ignore this warning. [ 504.639494] env[63273]: INFO nova.compute.manager [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Looking for unclaimed instances stuck in BUILDING status for nodes managed by this host [ 504.663637] env[63273]: WARNING nova.compute.manager [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] No compute node record found for host cpu-1. If this is the first time this service is starting on this host, then you can ignore this warning.: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host cpu-1 could not be found. [ 504.663871] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 504.664125] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 504.664291] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 504.664446] env[63273]: DEBUG nova.compute.resource_tracker [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 504.665809] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-179ded33-f521-4ab7-b757-085875096514 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.674858] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-796f233b-299a-4c62-8cd9-18adbeb870cb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.690252] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-117a4c66-e3f6-42b3-9525-01bf33091251 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.697372] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10b8258d-d368-4aa3-8bd4-35039e96facb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.726801] env[63273]: DEBUG nova.compute.resource_tracker [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180545MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 504.727038] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 504.727181] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 504.742361] env[63273]: WARNING nova.compute.resource_tracker [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] No compute node record for cpu-1:34a0ca39-8974-44d6-ab34-3ab6cf8432cd: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host 34a0ca39-8974-44d6-ab34-3ab6cf8432cd could not be found. [ 504.758376] env[63273]: INFO nova.compute.resource_tracker [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Compute node record created for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 with uuid: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd [ 504.820420] env[63273]: DEBUG nova.compute.resource_tracker [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 504.820607] env[63273]: DEBUG nova.compute.resource_tracker [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=100GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] stats={'failed_builds': '0'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 504.925685] env[63273]: INFO nova.scheduler.client.report [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] [req-9d26afdf-b3da-4499-8c86-523646c743df] Created resource provider record via placement API for resource provider with UUID 34a0ca39-8974-44d6-ab34-3ab6cf8432cd and name domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28. [ 504.944144] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b13e22a3-e698-4b48-a176-5bc5147dc051 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.951878] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e483321-327a-4e4b-81c9-5612fa91c750 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.983185] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7b70421-16e6-423f-8c25-e8d123a19a32 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 504.991268] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7aee4353-41a7-49fe-a95a-2649df1d462c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 505.004994] env[63273]: DEBUG nova.compute.provider_tree [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Updating inventory in ProviderTree for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 505.045826] env[63273]: DEBUG nova.scheduler.client.report [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Updated inventory for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd with generation 0 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:971}} [ 505.046095] env[63273]: DEBUG nova.compute.provider_tree [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Updating resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd generation from 0 to 1 during operation: update_inventory {{(pid=63273) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 505.046245] env[63273]: DEBUG nova.compute.provider_tree [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Updating inventory in ProviderTree for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 505.096658] env[63273]: DEBUG nova.compute.provider_tree [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Updating resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd generation from 1 to 2 during operation: update_traits {{(pid=63273) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 505.116309] env[63273]: DEBUG nova.compute.resource_tracker [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 505.116509] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.389s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 505.116668] env[63273]: DEBUG nova.service [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Creating RPC server for service compute {{(pid=63273) start /opt/stack/nova/nova/service.py:182}} [ 505.132250] env[63273]: DEBUG nova.service [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] Join ServiceGroup membership for this service compute {{(pid=63273) start /opt/stack/nova/nova/service.py:199}} [ 505.132446] env[63273]: DEBUG nova.servicegroup.drivers.db [None req-8ae07d61-dc07-44ca-a402-7ef67ae6c923 None None] DB_Driver: join new ServiceGroup member cpu-1 to the compute group, service = {{(pid=63273) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} [ 514.399679] env[63273]: DEBUG dbcounter [-] [63273] Writing DB stats nova_cell0:SELECT=1 {{(pid=63273) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 514.400527] env[63273]: DEBUG dbcounter [-] [63273] Writing DB stats nova_cell1:SELECT=1 {{(pid=63273) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 546.984734] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquiring lock "f948a055-fe10-4b55-b2e7-64f544c2d11a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 546.987025] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Lock "f948a055-fe10-4b55-b2e7-64f544c2d11a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 546.990401] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 546.990845] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.006360] env[63273]: DEBUG nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 547.010861] env[63273]: DEBUG nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 547.135778] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.136059] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.138024] env[63273]: INFO nova.compute.claims [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 547.141362] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.324806] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-699dfb53-fcb3-464e-8825-1ba1a235c4cb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.342526] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d846a0f9-149f-4a95-9959-b2eb15f5cf0f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.383840] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f4bb99e-6f84-4c42-8adb-a6c454d4a035 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.404027] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b96a5285-7f75-4096-a578-64410c1cf9a0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.419980] env[63273]: DEBUG nova.compute.provider_tree [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 547.436972] env[63273]: DEBUG nova.scheduler.client.report [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 547.474212] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.338s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.475522] env[63273]: DEBUG nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 547.478724] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.337s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.481729] env[63273]: INFO nova.compute.claims [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 547.576524] env[63273]: DEBUG nova.compute.utils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 547.582626] env[63273]: DEBUG nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 547.583246] env[63273]: DEBUG nova.network.neutron [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 547.633500] env[63273]: DEBUG nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 547.729015] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1f744da-6c06-45f2-84e5-da7a255de32a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.746701] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4369ff85-e617-4f57-868d-29b348456750 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.792930] env[63273]: DEBUG nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 547.794745] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef507498-7c29-45da-820c-ee643960ff86 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.805961] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fbe831e-e108-453e-b077-c6aa489e5158 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.822477] env[63273]: DEBUG nova.compute.provider_tree [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 547.842019] env[63273]: DEBUG nova.scheduler.client.report [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 547.870615] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.391s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.870615] env[63273]: DEBUG nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 547.945019] env[63273]: DEBUG nova.compute.utils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 547.946239] env[63273]: DEBUG nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Not allocating networking since 'none' was specified. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1969}} [ 547.967614] env[63273]: DEBUG nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 548.077967] env[63273]: DEBUG nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 548.112773] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquiring lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 548.113546] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 548.139411] env[63273]: DEBUG nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 548.221792] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 548.223496] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 548.224139] env[63273]: INFO nova.compute.claims [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 548.270857] env[63273]: DEBUG nova.virt.hardware [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 548.271110] env[63273]: DEBUG nova.virt.hardware [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 548.271270] env[63273]: DEBUG nova.virt.hardware [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 548.271838] env[63273]: DEBUG nova.virt.hardware [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 548.271921] env[63273]: DEBUG nova.virt.hardware [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 548.272175] env[63273]: DEBUG nova.virt.hardware [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 548.272614] env[63273]: DEBUG nova.virt.hardware [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 548.272614] env[63273]: DEBUG nova.virt.hardware [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 548.272926] env[63273]: DEBUG nova.virt.hardware [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 548.273098] env[63273]: DEBUG nova.virt.hardware [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 548.273275] env[63273]: DEBUG nova.virt.hardware [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 548.274318] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a7dab9b-794f-4175-bcec-6db54927633e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.289807] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9487c85e-76de-4644-99c0-d602ec57ef8c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.296608] env[63273]: DEBUG nova.virt.hardware [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 548.297636] env[63273]: DEBUG nova.virt.hardware [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 548.297636] env[63273]: DEBUG nova.virt.hardware [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 548.297636] env[63273]: DEBUG nova.virt.hardware [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 548.297636] env[63273]: DEBUG nova.virt.hardware [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 548.297636] env[63273]: DEBUG nova.virt.hardware [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 548.298244] env[63273]: DEBUG nova.virt.hardware [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 548.298563] env[63273]: DEBUG nova.virt.hardware [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 548.299364] env[63273]: DEBUG nova.virt.hardware [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 548.299644] env[63273]: DEBUG nova.virt.hardware [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 548.300691] env[63273]: DEBUG nova.virt.hardware [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 548.305299] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67fdaecb-82f6-4d1f-8264-b9054fcdf3f7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.332767] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccfff01d-8e6f-49b4-b18c-29eb9cc92aa8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.352737] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbb8d229-1ec9-41cd-aaa2-f132cad265c4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.375298] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Instance VIF info [] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 548.385518] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Creating folder: OpenStack. Parent ref: group-v4. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 548.389220] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d9aeb772-5040-46bc-863a-82f17c11fa3c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.405688] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Created folder: OpenStack in parent group-v4. [ 548.405969] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Creating folder: Project (ee937dfb6f0b4818b211ac74d11265ba). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 548.408784] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-4d064e6a-0ed0-4174-878e-934db5983bb5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.421758] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Created folder: Project (ee937dfb6f0b4818b211ac74d11265ba) in parent group-v986930. [ 548.422440] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Creating folder: Instances. Parent ref: group-v986931. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 548.422440] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-86facf69-e96b-41da-a63c-60e3034f3154 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.435792] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Created folder: Instances in parent group-v986931. [ 548.436463] env[63273]: DEBUG oslo.service.loopingcall [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 548.436463] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 548.436676] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3fedc1ea-30cf-4672-b632-e1cbb72c243b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.453038] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6402e5ff-2d2f-4406-8adf-7897fe3190d2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.459012] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 548.459012] env[63273]: value = "task-5071975" [ 548.459012] env[63273]: _type = "Task" [ 548.459012] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 548.465820] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d58c1423-f82e-49a3-afac-cbabfeea2aa8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.473610] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071975, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 548.505062] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d797bffd-c1b7-4da8-b848-299fa4f4d628 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.514472] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfe2b5df-94b9-4113-b0e1-15af56083249 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.530910] env[63273]: DEBUG nova.compute.provider_tree [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 548.557625] env[63273]: DEBUG nova.scheduler.client.report [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 548.590118] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.368s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 548.590708] env[63273]: DEBUG nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 548.649387] env[63273]: DEBUG nova.compute.utils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 548.663047] env[63273]: DEBUG nova.policy [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fc7b37355b3a4348908330557abb2e40', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6a04825b924a482994b31b7066af5714', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 548.672280] env[63273]: DEBUG nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 548.672875] env[63273]: DEBUG nova.network.neutron [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 548.701852] env[63273]: DEBUG nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 548.813741] env[63273]: DEBUG nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 548.861033] env[63273]: DEBUG nova.virt.hardware [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 548.866211] env[63273]: DEBUG nova.virt.hardware [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 548.866211] env[63273]: DEBUG nova.virt.hardware [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 548.866211] env[63273]: DEBUG nova.virt.hardware [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 548.866211] env[63273]: DEBUG nova.virt.hardware [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 548.866211] env[63273]: DEBUG nova.virt.hardware [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 548.866399] env[63273]: DEBUG nova.virt.hardware [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 548.867034] env[63273]: DEBUG nova.virt.hardware [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 548.867034] env[63273]: DEBUG nova.virt.hardware [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 548.867034] env[63273]: DEBUG nova.virt.hardware [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 548.867283] env[63273]: DEBUG nova.virt.hardware [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 548.868076] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f369fcac-d002-416b-af41-f74f3f68c89c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.876931] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef3f77a2-b76b-4f4e-b18d-575d708ea3ea {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.929526] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquiring lock "0952267f-c258-4575-be86-b2bc48e9863b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 548.929794] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Lock "0952267f-c258-4575-be86-b2bc48e9863b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 548.944762] env[63273]: DEBUG nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 548.971251] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071975, 'name': CreateVM_Task, 'duration_secs': 0.407962} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 548.971458] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 548.972758] env[63273]: DEBUG oslo_vmware.service [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8194b10a-1d46-40cc-8c44-3f9ae657cfdf {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.980753] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 548.981030] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 548.981836] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 548.982390] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b667579b-825b-4e3d-a00e-d5511b089a75 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.988796] env[63273]: DEBUG oslo_vmware.api [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Waiting for the task: (returnval){ [ 548.988796] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]527b4fa9-0bb1-310f-98cb-72e29c8926fc" [ 548.988796] env[63273]: _type = "Task" [ 548.988796] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 548.996999] env[63273]: DEBUG oslo_vmware.api [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]527b4fa9-0bb1-310f-98cb-72e29c8926fc, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 549.032461] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 549.032760] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 549.034647] env[63273]: INFO nova.compute.claims [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 549.171607] env[63273]: DEBUG nova.policy [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f22425d759e54279af47f4a2e3330a32', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '726f39967ec7455dadc4ed498eff094b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 549.206729] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f07d46a3-b8fc-4a55-9c01-2897558abf05 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.216814] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-135ba803-11e0-49ad-8483-c930c763eb4d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.265540] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b9f4046-d4b5-4034-8d03-c96cd1944766 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.278779] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7374ada-d02a-4032-bab4-3364aea826ab {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.296917] env[63273]: DEBUG nova.compute.provider_tree [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 549.316458] env[63273]: DEBUG nova.scheduler.client.report [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 549.347247] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.314s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 549.348717] env[63273]: DEBUG nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 549.412521] env[63273]: DEBUG nova.compute.utils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 549.414118] env[63273]: DEBUG nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Not allocating networking since 'none' was specified. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1969}} [ 549.427987] env[63273]: DEBUG nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 549.502349] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 549.502630] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 549.502864] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 549.503011] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 549.503621] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 549.503891] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e2866b90-9e50-4872-b6d0-a6c9433569b3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.515881] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 549.516083] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 549.516995] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3df3fcf9-3f01-42b8-aa6e-f56df7dbcd8c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.526851] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2e563987-266a-4aa2-aa79-54d88015dc7b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.533818] env[63273]: DEBUG oslo_vmware.api [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Waiting for the task: (returnval){ [ 549.533818] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e1bc0e-73a7-a30a-b60b-c1ce11341300" [ 549.533818] env[63273]: _type = "Task" [ 549.533818] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 549.539074] env[63273]: DEBUG nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 549.552799] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 549.552799] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Creating directory with path [datastore1] vmware_temp/9ee33103-217c-4e6c-8094-d3900a146f5a/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 549.552799] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-5b679078-f4b7-4809-b2d2-e8a1a7e6edad {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.580822] env[63273]: DEBUG nova.virt.hardware [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 549.581173] env[63273]: DEBUG nova.virt.hardware [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 549.581269] env[63273]: DEBUG nova.virt.hardware [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 549.581469] env[63273]: DEBUG nova.virt.hardware [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 549.581621] env[63273]: DEBUG nova.virt.hardware [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 549.581777] env[63273]: DEBUG nova.virt.hardware [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 549.581981] env[63273]: DEBUG nova.virt.hardware [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 549.582167] env[63273]: DEBUG nova.virt.hardware [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 549.582328] env[63273]: DEBUG nova.virt.hardware [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 549.582487] env[63273]: DEBUG nova.virt.hardware [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 549.582664] env[63273]: DEBUG nova.virt.hardware [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 549.583072] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Created directory with path [datastore1] vmware_temp/9ee33103-217c-4e6c-8094-d3900a146f5a/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 549.583219] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Fetch image to [datastore1] vmware_temp/9ee33103-217c-4e6c-8094-d3900a146f5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 549.583685] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/9ee33103-217c-4e6c-8094-d3900a146f5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 549.584840] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f2f30d7-24a9-42ce-b81f-56cd889e7b21 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.589255] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59385a15-1e94-43dd-a54c-e208238fa9d7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.600325] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ea03b79-aa64-41d8-b189-e7b14a18c211 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.604017] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6142b827-f0b9-4113-9630-20086546f52e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.638456] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dafd7a87-c497-479b-a500-fbe988468a94 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.643769] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Instance VIF info [] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 549.654774] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Creating folder: Project (c0deaad067fd40d684e8cecddd605788). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 549.654774] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-00f92e10-e8b7-4c76-a058-f55a12d35573 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.694249] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f417d8a4-a85a-4eec-841f-034e9e1a439e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.703597] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquiring lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 549.704365] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 549.711649] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-8a89b907-acc8-4204-b51f-b5849f554367 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.713900] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Created folder: Project (c0deaad067fd40d684e8cecddd605788) in parent group-v986930. [ 549.714103] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Creating folder: Instances. Parent ref: group-v986934. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 549.714960] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-05c55f15-38a3-4532-b01c-f959baf084e8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.725201] env[63273]: DEBUG nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 549.734019] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Created folder: Instances in parent group-v986934. [ 549.734019] env[63273]: DEBUG oslo.service.loopingcall [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 549.734019] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 549.734019] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-83806ff6-8d69-4635-af90-fcb5d9a741bb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.757712] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 549.757712] env[63273]: value = "task-5071978" [ 549.757712] env[63273]: _type = "Task" [ 549.757712] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 549.769788] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071978, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 549.823234] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 549.845573] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 549.847864] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 549.848789] env[63273]: INFO nova.compute.claims [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 549.972966] env[63273]: DEBUG oslo_vmware.rw_handles [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/9ee33103-217c-4e6c-8094-d3900a146f5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 550.062685] env[63273]: DEBUG oslo_vmware.rw_handles [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 550.063032] env[63273]: DEBUG oslo_vmware.rw_handles [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/9ee33103-217c-4e6c-8094-d3900a146f5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 550.183602] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c045b047-96b2-472f-814e-1973e82d1788 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.196355] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-860a09ea-f830-46c2-95cb-fe65467c63e5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.250158] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd1bf9a1-066a-41b2-9078-9065accd155d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.265017] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acec3fcc-2896-47ee-ad20-e7dc9fe28d48 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.278620] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071978, 'name': CreateVM_Task, 'duration_secs': 0.339278} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 550.290293] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 550.290293] env[63273]: DEBUG nova.compute.provider_tree [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 550.290293] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 550.290293] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 550.290293] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 550.290679] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b395f4fd-16fa-47b1-bbb1-9752486e2d95 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.295354] env[63273]: DEBUG oslo_vmware.api [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Waiting for the task: (returnval){ [ 550.295354] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]520a3898-ecbe-836c-3353-d5aec7660cfc" [ 550.295354] env[63273]: _type = "Task" [ 550.295354] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 550.300269] env[63273]: DEBUG nova.scheduler.client.report [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 550.311135] env[63273]: DEBUG oslo_vmware.api [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]520a3898-ecbe-836c-3353-d5aec7660cfc, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 550.561312] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.715s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 550.563048] env[63273]: DEBUG nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 550.655828] env[63273]: DEBUG nova.compute.utils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 550.660636] env[63273]: DEBUG nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 550.662705] env[63273]: DEBUG nova.network.neutron [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 550.678038] env[63273]: DEBUG nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 550.816906] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 550.817174] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 550.817383] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 550.818379] env[63273]: DEBUG nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 550.865627] env[63273]: DEBUG nova.virt.hardware [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 550.865937] env[63273]: DEBUG nova.virt.hardware [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 550.866061] env[63273]: DEBUG nova.virt.hardware [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 550.866484] env[63273]: DEBUG nova.virt.hardware [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 550.866484] env[63273]: DEBUG nova.virt.hardware [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 550.866595] env[63273]: DEBUG nova.virt.hardware [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 550.867213] env[63273]: DEBUG nova.virt.hardware [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 550.867213] env[63273]: DEBUG nova.virt.hardware [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 550.867213] env[63273]: DEBUG nova.virt.hardware [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 550.867213] env[63273]: DEBUG nova.virt.hardware [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 550.867400] env[63273]: DEBUG nova.virt.hardware [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 550.869021] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cc29325-ea13-4713-b67e-3a31f424ae91 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.881025] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95c6c318-9a00-4310-aade-303d0ba93629 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.368454] env[63273]: DEBUG nova.policy [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8d73259788e0405191360834be751d0b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '37d368bc456740739821f51e3a25f2fb', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 551.783995] env[63273]: DEBUG nova.network.neutron [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Successfully created port: 0ca9ef49-8de3-41cf-b25f-996926e56ea8 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 552.398424] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 552.400858] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.003s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 552.422897] env[63273]: DEBUG nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 552.517688] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 552.517850] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 552.519321] env[63273]: INFO nova.compute.claims [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 552.751417] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1321646-d9d0-4b10-acf5-d285a1abf2c5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.764551] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce0a1a88-7ed5-418a-89b1-451e3b157fcf {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.799904] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52cb5184-87fb-47b9-b2fe-95fd984e3844 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.809013] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa0c45a5-fed7-46e3-95ad-e000f61e8aa3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.827087] env[63273]: DEBUG nova.compute.provider_tree [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 552.840795] env[63273]: DEBUG nova.scheduler.client.report [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 552.863323] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.345s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 552.863820] env[63273]: DEBUG nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 552.929194] env[63273]: DEBUG nova.compute.utils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 552.930544] env[63273]: DEBUG nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 552.930727] env[63273]: DEBUG nova.network.neutron [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 552.949265] env[63273]: DEBUG nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 553.037570] env[63273]: DEBUG nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 553.057165] env[63273]: DEBUG nova.network.neutron [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Successfully created port: ed28b0e4-1a07-4893-9bb4-bcda73c61b70 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 553.077132] env[63273]: DEBUG nova.virt.hardware [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 553.077388] env[63273]: DEBUG nova.virt.hardware [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 553.077531] env[63273]: DEBUG nova.virt.hardware [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 553.077735] env[63273]: DEBUG nova.virt.hardware [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 553.077844] env[63273]: DEBUG nova.virt.hardware [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 553.077987] env[63273]: DEBUG nova.virt.hardware [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 553.079283] env[63273]: DEBUG nova.virt.hardware [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 553.079456] env[63273]: DEBUG nova.virt.hardware [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 553.079667] env[63273]: DEBUG nova.virt.hardware [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 553.079838] env[63273]: DEBUG nova.virt.hardware [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 553.080034] env[63273]: DEBUG nova.virt.hardware [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 553.081850] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a4c2328-b589-4401-8238-eb94f499dca2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.091778] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-257b2cea-bb31-446f-9247-2a58928f2457 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.447325] env[63273]: DEBUG nova.policy [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ba6c85eb49a749bbb96e8cc474f206bd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '418e6c5db17d4baf9c4f7cbc229ad07b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 553.896140] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Acquiring lock "11a32984-5548-4db4-8788-9b221ba381df" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 553.897599] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Lock "11a32984-5548-4db4-8788-9b221ba381df" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 553.916158] env[63273]: DEBUG nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 553.989042] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 553.989705] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 553.992151] env[63273]: INFO nova.compute.claims [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 554.247556] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96e3c32f-fb0c-4cf7-84ab-c4883680f5fd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.261423] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90c5463d-3580-4ceb-9b2b-03032c2900d2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.298023] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3015d76d-baa6-42a6-90f8-0b115e3bc15f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.306562] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2a85e9c-c826-467d-a059-887f6d3f8fb0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.322284] env[63273]: DEBUG nova.compute.provider_tree [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 554.335158] env[63273]: DEBUG nova.scheduler.client.report [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 554.359743] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.370s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.360315] env[63273]: DEBUG nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 554.413354] env[63273]: DEBUG nova.compute.utils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 554.415364] env[63273]: DEBUG nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 554.415685] env[63273]: DEBUG nova.network.neutron [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 554.427750] env[63273]: DEBUG nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 554.540651] env[63273]: DEBUG nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 554.582102] env[63273]: DEBUG nova.virt.hardware [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 554.582468] env[63273]: DEBUG nova.virt.hardware [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 554.582561] env[63273]: DEBUG nova.virt.hardware [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 554.582695] env[63273]: DEBUG nova.virt.hardware [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 554.582853] env[63273]: DEBUG nova.virt.hardware [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 554.583011] env[63273]: DEBUG nova.virt.hardware [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 554.585429] env[63273]: DEBUG nova.virt.hardware [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 554.585739] env[63273]: DEBUG nova.virt.hardware [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 554.585739] env[63273]: DEBUG nova.virt.hardware [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 554.585862] env[63273]: DEBUG nova.virt.hardware [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 554.586263] env[63273]: DEBUG nova.virt.hardware [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 554.587094] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc5a0d3c-c6ae-4d4b-95e3-3ba244986978 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.599759] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97ef876f-3c72-40df-96bf-01f87ce6c4d7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.718330] env[63273]: DEBUG nova.policy [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '07ad90bd59ef4e6bb7de0ffa39903cea', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '93e7b65b07c44f5c88154e6f398bdbd8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 554.894259] env[63273]: DEBUG nova.network.neutron [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Successfully created port: c479ac7d-7ab3-465a-8c20-ed211ff15840 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 556.618933] env[63273]: DEBUG nova.network.neutron [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Successfully created port: aaf2e5cf-5e39-426b-a2ce-a81327b9c403 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 557.076611] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquiring lock "a5c37718-0067-4f5e-ba7e-533f13529739" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 557.076611] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Lock "a5c37718-0067-4f5e-ba7e-533f13529739" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 557.101814] env[63273]: DEBUG nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 557.190456] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 557.190742] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.002s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 557.192278] env[63273]: INFO nova.compute.claims [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 557.506875] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c80bcbf-3340-4c81-ae6b-8863bf301169 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.518719] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-116760d3-8257-4fbc-bc18-425d8ecad655 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.558066] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ec3bea3-ffac-4858-8813-6d0435492a7c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.569084] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f4ced7d-56db-4e96-a629-6a296407f1dc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.596495] env[63273]: DEBUG nova.compute.provider_tree [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 557.612743] env[63273]: DEBUG nova.scheduler.client.report [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 557.653098] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.460s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 557.653098] env[63273]: DEBUG nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 557.718686] env[63273]: DEBUG nova.network.neutron [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Successfully created port: 25e8da26-340c-419f-8abe-a9e907a9f001 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 557.722627] env[63273]: DEBUG nova.compute.utils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 557.728279] env[63273]: DEBUG nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 557.728480] env[63273]: DEBUG nova.network.neutron [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 557.740707] env[63273]: DEBUG nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 557.839854] env[63273]: DEBUG nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 557.873680] env[63273]: DEBUG nova.virt.hardware [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 557.873971] env[63273]: DEBUG nova.virt.hardware [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 557.874134] env[63273]: DEBUG nova.virt.hardware [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 557.874296] env[63273]: DEBUG nova.virt.hardware [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 557.874441] env[63273]: DEBUG nova.virt.hardware [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 557.874581] env[63273]: DEBUG nova.virt.hardware [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 557.874786] env[63273]: DEBUG nova.virt.hardware [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 557.874939] env[63273]: DEBUG nova.virt.hardware [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 557.875149] env[63273]: DEBUG nova.virt.hardware [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 557.875318] env[63273]: DEBUG nova.virt.hardware [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 557.875488] env[63273]: DEBUG nova.virt.hardware [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 557.876394] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3cbccc7-8acc-4d46-ac7a-4fdd2a4c7d73 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.886174] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2593d3e1-0cf1-41af-93b5-7d0f2adf522d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 558.331829] env[63273]: DEBUG nova.policy [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '01b4d2cc262f487f94a03ff1c9331e0b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd806b4c5831b457fa075e387f55d8ad5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 558.760848] env[63273]: DEBUG nova.network.neutron [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Successfully updated port: 0ca9ef49-8de3-41cf-b25f-996926e56ea8 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 558.777586] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "refresh_cache-4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 558.777783] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquired lock "refresh_cache-4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 558.777987] env[63273]: DEBUG nova.network.neutron [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 559.080508] env[63273]: DEBUG nova.network.neutron [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 559.378147] env[63273]: DEBUG nova.network.neutron [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Successfully updated port: ed28b0e4-1a07-4893-9bb4-bcda73c61b70 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 559.400176] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquiring lock "refresh_cache-02ed8dca-f6ae-47b0-a047-f89f858f5ff2" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 559.400343] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquired lock "refresh_cache-02ed8dca-f6ae-47b0-a047-f89f858f5ff2" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 559.400535] env[63273]: DEBUG nova.network.neutron [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 559.715552] env[63273]: DEBUG nova.network.neutron [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 559.877029] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 559.878789] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 559.892531] env[63273]: DEBUG nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 559.972560] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 559.973026] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 559.978929] env[63273]: INFO nova.compute.claims [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 560.275165] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b68bc43a-e29d-4045-ba08-1d4640903ac0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 560.285559] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-394d9127-1b5c-42f4-9df2-16a63a6b677e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 560.333829] env[63273]: DEBUG nova.network.neutron [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Successfully updated port: aaf2e5cf-5e39-426b-a2ce-a81327b9c403 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 560.340330] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-869c4dc9-99f2-42f2-a596-b4362ecc9dc3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 560.354171] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55538c91-fa2e-4d9a-8341-40c3b308420c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 560.359079] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Acquiring lock "refresh_cache-11a32984-5548-4db4-8788-9b221ba381df" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 560.359079] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Acquired lock "refresh_cache-11a32984-5548-4db4-8788-9b221ba381df" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 560.359079] env[63273]: DEBUG nova.network.neutron [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 560.378876] env[63273]: DEBUG nova.compute.provider_tree [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 560.396092] env[63273]: DEBUG nova.scheduler.client.report [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 560.419309] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.445s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 560.419309] env[63273]: DEBUG nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 560.499473] env[63273]: DEBUG nova.compute.utils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 560.501113] env[63273]: DEBUG nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 560.501261] env[63273]: DEBUG nova.network.neutron [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 560.518926] env[63273]: DEBUG nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 560.585426] env[63273]: DEBUG nova.network.neutron [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 560.649828] env[63273]: DEBUG nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 560.693876] env[63273]: DEBUG nova.virt.hardware [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 560.695713] env[63273]: DEBUG nova.virt.hardware [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 560.695713] env[63273]: DEBUG nova.virt.hardware [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 560.695713] env[63273]: DEBUG nova.virt.hardware [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 560.695713] env[63273]: DEBUG nova.virt.hardware [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 560.695713] env[63273]: DEBUG nova.virt.hardware [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 560.696077] env[63273]: DEBUG nova.virt.hardware [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 560.696077] env[63273]: DEBUG nova.virt.hardware [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 560.696077] env[63273]: DEBUG nova.virt.hardware [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 560.696077] env[63273]: DEBUG nova.virt.hardware [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 560.696077] env[63273]: DEBUG nova.virt.hardware [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 560.697022] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-850accf9-c7fd-4962-8ed7-eb1401ab69bc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 560.710178] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f72c10e5-0a0c-43d1-8158-bb186c375b63 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 560.831407] env[63273]: DEBUG nova.policy [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7648b8a75224424f8226b574da373a87', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '495ae26325184812a744fc95da67038b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 561.452390] env[63273]: DEBUG nova.network.neutron [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Updating instance_info_cache with network_info: [{"id": "0ca9ef49-8de3-41cf-b25f-996926e56ea8", "address": "fa:16:3e:0f:40:a6", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.173", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap0ca9ef49-8d", "ovs_interfaceid": "0ca9ef49-8de3-41cf-b25f-996926e56ea8", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 561.477325] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Releasing lock "refresh_cache-4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 561.477325] env[63273]: DEBUG nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Instance network_info: |[{"id": "0ca9ef49-8de3-41cf-b25f-996926e56ea8", "address": "fa:16:3e:0f:40:a6", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.173", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap0ca9ef49-8d", "ovs_interfaceid": "0ca9ef49-8de3-41cf-b25f-996926e56ea8", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 561.477554] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:0f:40:a6', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f66f8375-4460-4acd-987b-acda72bfcf0d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '0ca9ef49-8de3-41cf-b25f-996926e56ea8', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 561.486519] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Creating folder: Project (6a04825b924a482994b31b7066af5714). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 561.487792] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-94e229d2-f994-449d-a977-c06e2acc1239 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.501562] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Created folder: Project (6a04825b924a482994b31b7066af5714) in parent group-v986930. [ 561.501961] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Creating folder: Instances. Parent ref: group-v986937. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 561.503031] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-26b4771d-f402-4ad3-92e2-ee0dbc3d4043 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.515209] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Created folder: Instances in parent group-v986937. [ 561.515579] env[63273]: DEBUG oslo.service.loopingcall [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 561.515856] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 561.516132] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-0d0b0eaf-78fb-4592-8985-d0e8e3df7b83 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.541812] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 561.541812] env[63273]: value = "task-5071981" [ 561.541812] env[63273]: _type = "Task" [ 561.541812] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 561.553284] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071981, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 561.791551] env[63273]: DEBUG nova.network.neutron [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Updating instance_info_cache with network_info: [{"id": "aaf2e5cf-5e39-426b-a2ce-a81327b9c403", "address": "fa:16:3e:80:fa:32", "network": {"id": "c29ef6fe-282c-4f3f-b816-280b529e8985", "bridge": "br-int", "label": "tempest-ImagesOneServerNegativeTestJSON-477898296-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "93e7b65b07c44f5c88154e6f398bdbd8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "21310d90-efbc-45a8-a97f-c4358606530f", "external-id": "nsx-vlan-transportzone-672", "segmentation_id": 672, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapaaf2e5cf-5e", "ovs_interfaceid": "aaf2e5cf-5e39-426b-a2ce-a81327b9c403", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 561.821159] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Releasing lock "refresh_cache-11a32984-5548-4db4-8788-9b221ba381df" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 561.821466] env[63273]: DEBUG nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Instance network_info: |[{"id": "aaf2e5cf-5e39-426b-a2ce-a81327b9c403", "address": "fa:16:3e:80:fa:32", "network": {"id": "c29ef6fe-282c-4f3f-b816-280b529e8985", "bridge": "br-int", "label": "tempest-ImagesOneServerNegativeTestJSON-477898296-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "93e7b65b07c44f5c88154e6f398bdbd8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "21310d90-efbc-45a8-a97f-c4358606530f", "external-id": "nsx-vlan-transportzone-672", "segmentation_id": 672, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapaaf2e5cf-5e", "ovs_interfaceid": "aaf2e5cf-5e39-426b-a2ce-a81327b9c403", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 561.821952] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:80:fa:32', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '21310d90-efbc-45a8-a97f-c4358606530f', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'aaf2e5cf-5e39-426b-a2ce-a81327b9c403', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 561.837438] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Creating folder: Project (93e7b65b07c44f5c88154e6f398bdbd8). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 561.837438] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-16cddea6-4e3e-4611-9b63-25fa56ba9140 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.850102] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Created folder: Project (93e7b65b07c44f5c88154e6f398bdbd8) in parent group-v986930. [ 561.850102] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Creating folder: Instances. Parent ref: group-v986940. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 561.850238] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-f54b510c-a0f5-4fff-aa9c-f28b2a6a1835 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.865652] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Created folder: Instances in parent group-v986940. [ 561.865933] env[63273]: DEBUG oslo.service.loopingcall [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 561.866167] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 561.866399] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-be98dbb3-a9ae-45c6-887f-71b39f5ed13c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.892664] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 561.892664] env[63273]: value = "task-5071984" [ 561.892664] env[63273]: _type = "Task" [ 561.892664] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 561.906802] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071984, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 562.055743] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071981, 'name': CreateVM_Task, 'duration_secs': 0.38033} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 562.056201] env[63273]: DEBUG nova.network.neutron [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Successfully updated port: c479ac7d-7ab3-465a-8c20-ed211ff15840 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 562.057351] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 562.069176] env[63273]: DEBUG nova.network.neutron [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Updating instance_info_cache with network_info: [{"id": "ed28b0e4-1a07-4893-9bb4-bcda73c61b70", "address": "fa:16:3e:32:98:f3", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.180", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "taped28b0e4-1a", "ovs_interfaceid": "ed28b0e4-1a07-4893-9bb4-bcda73c61b70", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 562.083181] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquiring lock "refresh_cache-f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 562.083350] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquired lock "refresh_cache-f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 562.083606] env[63273]: DEBUG nova.network.neutron [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 562.108725] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Releasing lock "refresh_cache-02ed8dca-f6ae-47b0-a047-f89f858f5ff2" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 562.108725] env[63273]: DEBUG nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Instance network_info: |[{"id": "ed28b0e4-1a07-4893-9bb4-bcda73c61b70", "address": "fa:16:3e:32:98:f3", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.180", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "taped28b0e4-1a", "ovs_interfaceid": "ed28b0e4-1a07-4893-9bb4-bcda73c61b70", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 562.108841] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:32:98:f3', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f66f8375-4460-4acd-987b-acda72bfcf0d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'ed28b0e4-1a07-4893-9bb4-bcda73c61b70', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 562.114853] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Creating folder: Project (726f39967ec7455dadc4ed498eff094b). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 562.122238] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-16aef3f2-f419-473b-90ce-b782c404520d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.126136] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquiring lock "94d19d7f-8eb7-4c93-aff4-3b1404338240" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 562.126748] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Lock "94d19d7f-8eb7-4c93-aff4-3b1404338240" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.140699] env[63273]: DEBUG nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 562.147018] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Created folder: Project (726f39967ec7455dadc4ed498eff094b) in parent group-v986930. [ 562.147018] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Creating folder: Instances. Parent ref: group-v986943. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 562.147018] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-af35ea48-3346-4ea8-b47b-4ec5e54c5009 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.158461] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Created folder: Instances in parent group-v986943. [ 562.159572] env[63273]: DEBUG oslo.service.loopingcall [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 562.160326] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 562.160544] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 562.160893] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 562.161193] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 562.161385] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-890a4265-be98-4c3a-b5e7-248d14717907 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.163657] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3fa7289c-62e3-4806-b46e-7eecd8d4693d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.182778] env[63273]: DEBUG nova.network.neutron [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Successfully created port: 65bfc0ed-f2b0-48d3-a0cf-025b076c69ce {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 562.193115] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 562.193115] env[63273]: value = "task-5071987" [ 562.193115] env[63273]: _type = "Task" [ 562.193115] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 562.194938] env[63273]: DEBUG oslo_vmware.api [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Waiting for the task: (returnval){ [ 562.194938] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]520e4dfd-b8f2-19ae-cc63-27bca896ebe0" [ 562.194938] env[63273]: _type = "Task" [ 562.194938] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 562.222107] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071987, 'name': CreateVM_Task} progress is 5%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 562.222559] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 562.223726] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 562.224049] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 562.243302] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 562.243380] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.250356] env[63273]: INFO nova.compute.claims [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 562.408481] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071984, 'name': CreateVM_Task, 'duration_secs': 0.430544} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 562.408481] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 562.408481] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 562.408481] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 562.409231] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 562.412342] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-3649c990-3064-4693-81be-bd5a6261f6df {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.415854] env[63273]: DEBUG nova.network.neutron [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Successfully created port: 989e529e-348e-496a-8c6e-be4e8530306a {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 562.423691] env[63273]: DEBUG oslo_vmware.api [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Waiting for the task: (returnval){ [ 562.423691] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ca5fa3-c71f-9aea-86aa-9c7ce0b32d5e" [ 562.423691] env[63273]: _type = "Task" [ 562.423691] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 562.437337] env[63273]: DEBUG oslo_vmware.api [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ca5fa3-c71f-9aea-86aa-9c7ce0b32d5e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 562.442392] env[63273]: DEBUG nova.network.neutron [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 562.612366] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5832d2d-5025-4bac-aa03-d5f122170172 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.626011] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-305eaf82-5fc5-45f1-9220-8e9092d90284 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.666079] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-118aaf41-2384-4be2-a67b-2fea9d7aadd9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.679468] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ce6f8a1-d243-4d7d-a83d-b04a612f694e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.702575] env[63273]: DEBUG nova.compute.provider_tree [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 562.715564] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071987, 'name': CreateVM_Task} progress is 99%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 562.721880] env[63273]: DEBUG nova.scheduler.client.report [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 562.750414] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.507s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 562.750995] env[63273]: DEBUG nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 562.811394] env[63273]: DEBUG nova.compute.utils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 562.813741] env[63273]: DEBUG nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 562.813965] env[63273]: DEBUG nova.network.neutron [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 562.839631] env[63273]: DEBUG nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 562.850318] env[63273]: DEBUG nova.compute.manager [req-63a2ab2f-0438-4c34-98a2-2d9a277abfad req-34db2b74-3c2e-47e7-b41b-18076888cac7 service nova] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Received event network-vif-plugged-0ca9ef49-8de3-41cf-b25f-996926e56ea8 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 562.850557] env[63273]: DEBUG oslo_concurrency.lockutils [req-63a2ab2f-0438-4c34-98a2-2d9a277abfad req-34db2b74-3c2e-47e7-b41b-18076888cac7 service nova] Acquiring lock "4afbddcb-d9ac-412e-93d9-52b2de6fbdb5-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 562.850782] env[63273]: DEBUG oslo_concurrency.lockutils [req-63a2ab2f-0438-4c34-98a2-2d9a277abfad req-34db2b74-3c2e-47e7-b41b-18076888cac7 service nova] Lock "4afbddcb-d9ac-412e-93d9-52b2de6fbdb5-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.850955] env[63273]: DEBUG oslo_concurrency.lockutils [req-63a2ab2f-0438-4c34-98a2-2d9a277abfad req-34db2b74-3c2e-47e7-b41b-18076888cac7 service nova] Lock "4afbddcb-d9ac-412e-93d9-52b2de6fbdb5-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 562.851127] env[63273]: DEBUG nova.compute.manager [req-63a2ab2f-0438-4c34-98a2-2d9a277abfad req-34db2b74-3c2e-47e7-b41b-18076888cac7 service nova] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] No waiting events found dispatching network-vif-plugged-0ca9ef49-8de3-41cf-b25f-996926e56ea8 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 562.852255] env[63273]: WARNING nova.compute.manager [req-63a2ab2f-0438-4c34-98a2-2d9a277abfad req-34db2b74-3c2e-47e7-b41b-18076888cac7 service nova] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Received unexpected event network-vif-plugged-0ca9ef49-8de3-41cf-b25f-996926e56ea8 for instance with vm_state building and task_state spawning. [ 562.933658] env[63273]: DEBUG nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 562.940560] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 562.941203] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 562.941203] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 562.966990] env[63273]: DEBUG nova.virt.hardware [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 562.967754] env[63273]: DEBUG nova.virt.hardware [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 562.967754] env[63273]: DEBUG nova.virt.hardware [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 562.967754] env[63273]: DEBUG nova.virt.hardware [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 562.967754] env[63273]: DEBUG nova.virt.hardware [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 562.967941] env[63273]: DEBUG nova.virt.hardware [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 562.968610] env[63273]: DEBUG nova.virt.hardware [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 562.968856] env[63273]: DEBUG nova.virt.hardware [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 562.969061] env[63273]: DEBUG nova.virt.hardware [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 562.969240] env[63273]: DEBUG nova.virt.hardware [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 562.969463] env[63273]: DEBUG nova.virt.hardware [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 562.970307] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-153f38cc-75e2-40ad-84e9-fab9b3753357 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.982164] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3936da88-1873-469a-8f2e-6bab2f859b79 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 563.217878] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071987, 'name': CreateVM_Task} progress is 99%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 563.500334] env[63273]: DEBUG nova.policy [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f2c36e01c1a0431eb6c84dd372012c37', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '4b844f0a784d4f20a5adbf7354c75cc7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 563.713855] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071987, 'name': CreateVM_Task, 'duration_secs': 1.366493} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 563.714203] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 563.714776] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 563.715107] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 563.715855] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 563.715855] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8e2c1453-8f54-4d2f-b138-7978b5171828 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 563.722404] env[63273]: DEBUG oslo_vmware.api [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Waiting for the task: (returnval){ [ 563.722404] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]521e61b5-36d9-302f-3559-2c3f7ec8564d" [ 563.722404] env[63273]: _type = "Task" [ 563.722404] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 563.730334] env[63273]: DEBUG oslo_vmware.api [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]521e61b5-36d9-302f-3559-2c3f7ec8564d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 564.050321] env[63273]: DEBUG nova.network.neutron [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Updating instance_info_cache with network_info: [{"id": "c479ac7d-7ab3-465a-8c20-ed211ff15840", "address": "fa:16:3e:b1:90:48", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.65", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc479ac7d-7a", "ovs_interfaceid": "c479ac7d-7ab3-465a-8c20-ed211ff15840", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 564.063827] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Releasing lock "refresh_cache-f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 564.064411] env[63273]: DEBUG nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Instance network_info: |[{"id": "c479ac7d-7ab3-465a-8c20-ed211ff15840", "address": "fa:16:3e:b1:90:48", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.65", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc479ac7d-7a", "ovs_interfaceid": "c479ac7d-7ab3-465a-8c20-ed211ff15840", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 564.064586] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:b1:90:48', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f66f8375-4460-4acd-987b-acda72bfcf0d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'c479ac7d-7ab3-465a-8c20-ed211ff15840', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 564.077381] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Creating folder: Project (37d368bc456740739821f51e3a25f2fb). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 564.078362] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-078016d9-cbbd-49cb-8b15-818c26bf46ba {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.095848] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Created folder: Project (37d368bc456740739821f51e3a25f2fb) in parent group-v986930. [ 564.096205] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Creating folder: Instances. Parent ref: group-v986946. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 564.097967] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-47e324f7-8fe6-4f14-965a-0f9ba45ad33b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.112021] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Created folder: Instances in parent group-v986946. [ 564.112021] env[63273]: DEBUG oslo.service.loopingcall [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 564.112021] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 564.112021] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-2aca8654-8561-405f-b3a1-67431d41f624 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.134124] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 564.135814] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 564.136045] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 564.136269] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 564.138806] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 564.138806] env[63273]: value = "task-5071990" [ 564.138806] env[63273]: _type = "Task" [ 564.138806] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 564.152898] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071990, 'name': CreateVM_Task} progress is 5%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 564.169298] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 564.169608] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 564.169736] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 564.169879] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 564.170034] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 564.171171] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 564.171359] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 564.172839] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 564.174034] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 564.174034] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 564.174034] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 564.175653] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 564.175653] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 564.175653] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 564.175653] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 564.175653] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 564.175653] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_power_states {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 564.196816] env[63273]: DEBUG nova.network.neutron [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Successfully updated port: 65bfc0ed-f2b0-48d3-a0cf-025b076c69ce {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 564.217034] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Getting list of instances from cluster (obj){ [ 564.217034] env[63273]: value = "domain-c8" [ 564.217034] env[63273]: _type = "ClusterComputeResource" [ 564.217034] env[63273]: } {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 564.224293] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b87ac56a-5a0b-428a-a907-5a5e250a60e0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.230514] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "refresh_cache-ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 564.233032] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "refresh_cache-ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 564.233032] env[63273]: DEBUG nova.network.neutron [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 564.265207] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Got total of 5 instances {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 564.265770] env[63273]: WARNING nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] While synchronizing instance power states, found 10 instances in the database and 5 instances on the hypervisor. [ 564.265770] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 564.268198] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid f948a055-fe10-4b55-b2e7-64f544c2d11a {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 564.268401] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 02ed8dca-f6ae-47b0-a047-f89f858f5ff2 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 564.268619] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 0952267f-c258-4575-be86-b2bc48e9863b {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 564.268793] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid f1dddee3-33d6-4f55-bd91-9ce2fe6be72b {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 564.268998] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 564.269180] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 11a32984-5548-4db4-8788-9b221ba381df {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 564.269336] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid a5c37718-0067-4f5e-ba7e-533f13529739 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 564.269503] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid ab9cf87a-b74f-4b2f-842d-26e16b7fdb92 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 564.269659] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 94d19d7f-8eb7-4c93-aff4-3b1404338240 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 564.276564] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.276564] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "f948a055-fe10-4b55-b2e7-64f544c2d11a" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.276564] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.276564] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "0952267f-c258-4575-be86-b2bc48e9863b" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.277331] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.277331] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.277331] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "11a32984-5548-4db4-8788-9b221ba381df" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.277331] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "a5c37718-0067-4f5e-ba7e-533f13529739" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.277557] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.277557] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "94d19d7f-8eb7-4c93-aff4-3b1404338240" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.277557] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 564.277650] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 564.277870] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 564.279438] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 564.279438] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 564.279583] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 564.313468] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.313468] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 564.313468] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 564.313468] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 564.315738] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f3a7875-696b-4a52-9ddc-5e471b4d1811 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.336235] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad6278e0-fb5e-45b9-bdad-211e9c66cd5f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.354675] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c79ffb37-f149-4d98-b4cc-40a01901523c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.365162] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99ca387d-5927-4f6a-8178-0f1fd90a193a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.403544] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180537MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 564.403675] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.403891] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 564.486822] env[63273]: DEBUG nova.network.neutron [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 564.537528] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 564.537528] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f948a055-fe10-4b55-b2e7-64f544c2d11a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 564.537528] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 02ed8dca-f6ae-47b0-a047-f89f858f5ff2 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 564.537528] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0952267f-c258-4575-be86-b2bc48e9863b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 564.538224] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f1dddee3-33d6-4f55-bd91-9ce2fe6be72b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 564.538224] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 564.538224] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 11a32984-5548-4db4-8788-9b221ba381df actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 564.538224] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a5c37718-0067-4f5e-ba7e-533f13529739 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 564.538355] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance ab9cf87a-b74f-4b2f-842d-26e16b7fdb92 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 564.538355] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 94d19d7f-8eb7-4c93-aff4-3b1404338240 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 564.538355] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 564.540983] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '0', 'num_instances': '10', 'num_vm_building': '10', 'num_task_spawning': '10', 'num_os_type_None': '10', 'num_proj_6a04825b924a482994b31b7066af5714': '1', 'io_workload': '10', 'num_proj_ee937dfb6f0b4818b211ac74d11265ba': '1', 'num_proj_726f39967ec7455dadc4ed498eff094b': '1', 'num_proj_c0deaad067fd40d684e8cecddd605788': '1', 'num_proj_37d368bc456740739821f51e3a25f2fb': '1', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'num_proj_93e7b65b07c44f5c88154e6f398bdbd8': '1', 'num_proj_d806b4c5831b457fa075e387f55d8ad5': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_4b844f0a784d4f20a5adbf7354c75cc7': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 564.656920] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071990, 'name': CreateVM_Task} progress is 99%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 564.774173] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquiring lock "5501c192-867b-40c0-9bea-c3c44865d2f4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.775304] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Lock "5501c192-867b-40c0-9bea-c3c44865d2f4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 564.874094] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40795b65-6344-4700-8274-a4a6507b1e67 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.885816] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f43dbbd-9c99-4845-ab5b-935f260c3266 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.930590] env[63273]: DEBUG nova.network.neutron [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Successfully updated port: 25e8da26-340c-419f-8abe-a9e907a9f001 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 564.934725] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a359517d-ab92-4c14-b411-3c838a32504b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.942371] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1252380-265c-4f99-bd0c-843284ec27d7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.950643] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "refresh_cache-89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 564.950643] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquired lock "refresh_cache-89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 564.950643] env[63273]: DEBUG nova.network.neutron [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 564.973491] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 564.986492] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 565.013022] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 565.013022] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.605s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 565.013022] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_running_deleted_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 565.013022] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Getting list of instances from cluster (obj){ [ 565.013022] env[63273]: value = "domain-c8" [ 565.013022] env[63273]: _type = "ClusterComputeResource" [ 565.013022] env[63273]: } {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 565.013022] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-337a1695-e881-41ba-a0cf-c628de3064c7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 565.027281] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Got total of 5 instances {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 565.153460] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071990, 'name': CreateVM_Task} progress is 99%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 565.157136] env[63273]: DEBUG nova.network.neutron [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 565.215619] env[63273]: DEBUG nova.compute.manager [req-ec843051-28c0-4154-8fcc-6bc95bd6df13 req-ce377d6a-b906-42ca-98a5-98035077a188 service nova] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Received event network-vif-plugged-aaf2e5cf-5e39-426b-a2ce-a81327b9c403 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 565.215619] env[63273]: DEBUG oslo_concurrency.lockutils [req-ec843051-28c0-4154-8fcc-6bc95bd6df13 req-ce377d6a-b906-42ca-98a5-98035077a188 service nova] Acquiring lock "11a32984-5548-4db4-8788-9b221ba381df-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 565.215943] env[63273]: DEBUG oslo_concurrency.lockutils [req-ec843051-28c0-4154-8fcc-6bc95bd6df13 req-ce377d6a-b906-42ca-98a5-98035077a188 service nova] Lock "11a32984-5548-4db4-8788-9b221ba381df-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 565.215943] env[63273]: DEBUG oslo_concurrency.lockutils [req-ec843051-28c0-4154-8fcc-6bc95bd6df13 req-ce377d6a-b906-42ca-98a5-98035077a188 service nova] Lock "11a32984-5548-4db4-8788-9b221ba381df-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 565.216234] env[63273]: DEBUG nova.compute.manager [req-ec843051-28c0-4154-8fcc-6bc95bd6df13 req-ce377d6a-b906-42ca-98a5-98035077a188 service nova] [instance: 11a32984-5548-4db4-8788-9b221ba381df] No waiting events found dispatching network-vif-plugged-aaf2e5cf-5e39-426b-a2ce-a81327b9c403 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 565.216425] env[63273]: WARNING nova.compute.manager [req-ec843051-28c0-4154-8fcc-6bc95bd6df13 req-ce377d6a-b906-42ca-98a5-98035077a188 service nova] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Received unexpected event network-vif-plugged-aaf2e5cf-5e39-426b-a2ce-a81327b9c403 for instance with vm_state building and task_state spawning. [ 565.219157] env[63273]: DEBUG nova.network.neutron [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Updating instance_info_cache with network_info: [{"id": "65bfc0ed-f2b0-48d3-a0cf-025b076c69ce", "address": "fa:16:3e:5a:d3:50", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap65bfc0ed-f2", "ovs_interfaceid": "65bfc0ed-f2b0-48d3-a0cf-025b076c69ce", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 565.246418] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "refresh_cache-ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 565.248234] env[63273]: DEBUG nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Instance network_info: |[{"id": "65bfc0ed-f2b0-48d3-a0cf-025b076c69ce", "address": "fa:16:3e:5a:d3:50", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap65bfc0ed-f2", "ovs_interfaceid": "65bfc0ed-f2b0-48d3-a0cf-025b076c69ce", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 565.248408] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:5a:d3:50', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '69e1a5c1-4ae7-409b-8de7-d401684ef60d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '65bfc0ed-f2b0-48d3-a0cf-025b076c69ce', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 565.258909] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating folder: Project (495ae26325184812a744fc95da67038b). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 565.263206] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d512511e-e2e0-41c5-864d-b8c98f24e4d1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 565.278262] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created folder: Project (495ae26325184812a744fc95da67038b) in parent group-v986930. [ 565.278788] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating folder: Instances. Parent ref: group-v986952. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 565.278788] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-1bf04e66-f61e-4eda-b65d-c90f4ba99d9e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 565.295197] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created folder: Instances in parent group-v986952. [ 565.298983] env[63273]: DEBUG oslo.service.loopingcall [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 565.298983] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 565.298983] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-f8aa6bbe-a9f1-4868-ad9b-3bdc08f1219a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 565.323843] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 565.323843] env[63273]: value = "task-5071997" [ 565.323843] env[63273]: _type = "Task" [ 565.323843] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 565.334457] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071997, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 565.650146] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071990, 'name': CreateVM_Task, 'duration_secs': 1.455196} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 565.651096] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 565.651096] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 565.651243] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 565.651591] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 565.651822] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c0a85909-b7e9-4f0f-94a9-63ac4f3841ad {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 565.657078] env[63273]: DEBUG oslo_vmware.api [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Waiting for the task: (returnval){ [ 565.657078] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]528291d4-1a54-dd9e-09cc-093370cafbfc" [ 565.657078] env[63273]: _type = "Task" [ 565.657078] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 565.666014] env[63273]: DEBUG oslo_vmware.api [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]528291d4-1a54-dd9e-09cc-093370cafbfc, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 565.842028] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071997, 'name': CreateVM_Task} progress is 99%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 566.144353] env[63273]: DEBUG nova.network.neutron [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Successfully created port: e7c7fd55-2386-484e-b08a-2ce6f4da39fc {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 566.174908] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 566.175677] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 566.176368] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 566.337289] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5071997, 'name': CreateVM_Task, 'duration_secs': 0.571891} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 566.338350] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 566.340023] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 566.340023] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 566.340023] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 566.340023] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-af07a2fd-5689-44d7-869c-cea19e9d5280 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 566.349182] env[63273]: DEBUG oslo_vmware.api [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 566.349182] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5251b795-f4fd-37ca-bdc2-994c28846a86" [ 566.349182] env[63273]: _type = "Task" [ 566.349182] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 566.358235] env[63273]: DEBUG oslo_vmware.api [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5251b795-f4fd-37ca-bdc2-994c28846a86, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 566.377311] env[63273]: DEBUG nova.network.neutron [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Updating instance_info_cache with network_info: [{"id": "25e8da26-340c-419f-8abe-a9e907a9f001", "address": "fa:16:3e:a7:e0:c6", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.183", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap25e8da26-34", "ovs_interfaceid": "25e8da26-340c-419f-8abe-a9e907a9f001", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 566.394360] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Releasing lock "refresh_cache-89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 566.394760] env[63273]: DEBUG nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Instance network_info: |[{"id": "25e8da26-340c-419f-8abe-a9e907a9f001", "address": "fa:16:3e:a7:e0:c6", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.183", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap25e8da26-34", "ovs_interfaceid": "25e8da26-340c-419f-8abe-a9e907a9f001", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 566.395397] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:a7:e0:c6', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f66f8375-4460-4acd-987b-acda72bfcf0d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '25e8da26-340c-419f-8abe-a9e907a9f001', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 566.403278] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Creating folder: Project (418e6c5db17d4baf9c4f7cbc229ad07b). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 566.403953] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-a992e8ae-a48f-467b-8704-86d2b6c05924 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 566.417721] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Created folder: Project (418e6c5db17d4baf9c4f7cbc229ad07b) in parent group-v986930. [ 566.417938] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Creating folder: Instances. Parent ref: group-v986955. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 566.418236] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-0c95dc6e-5162-4f9c-ab46-50c5fba53aa8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 566.431952] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Created folder: Instances in parent group-v986955. [ 566.432331] env[63273]: DEBUG oslo.service.loopingcall [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 566.432634] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 566.432879] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-03a5cd5b-2bf3-4593-91d7-8dbf30c53a8d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 566.456318] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 566.456318] env[63273]: value = "task-5072000" [ 566.456318] env[63273]: _type = "Task" [ 566.456318] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 566.466506] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072000, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 566.867758] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 566.869844] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 566.869844] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 566.953548] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquiring lock "77493b32-817c-4be8-a42d-8d48e707d41f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 566.953548] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Lock "77493b32-817c-4be8-a42d-8d48e707d41f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 566.970484] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072000, 'name': CreateVM_Task} progress is 99%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 567.354388] env[63273]: DEBUG nova.network.neutron [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Successfully updated port: 989e529e-348e-496a-8c6e-be4e8530306a {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 567.364946] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquiring lock "refresh_cache-a5c37718-0067-4f5e-ba7e-533f13529739" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 567.365119] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquired lock "refresh_cache-a5c37718-0067-4f5e-ba7e-533f13529739" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 567.365272] env[63273]: DEBUG nova.network.neutron [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 567.476456] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072000, 'name': CreateVM_Task, 'duration_secs': 0.581108} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 567.478685] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 567.478685] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 567.478685] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 567.478685] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 567.478685] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-e7c3bdc0-1240-4335-b8e0-f25099f89da1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 567.489280] env[63273]: DEBUG oslo_vmware.api [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Waiting for the task: (returnval){ [ 567.489280] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ad0c1e-be7c-5a1d-e9bb-0e697474a4e3" [ 567.489280] env[63273]: _type = "Task" [ 567.489280] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 567.502734] env[63273]: DEBUG oslo_vmware.api [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ad0c1e-be7c-5a1d-e9bb-0e697474a4e3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 567.531482] env[63273]: DEBUG nova.network.neutron [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 567.568114] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Received event network-vif-plugged-ed28b0e4-1a07-4893-9bb4-bcda73c61b70 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 567.568114] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Acquiring lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 567.568114] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 567.568114] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 567.568277] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] No waiting events found dispatching network-vif-plugged-ed28b0e4-1a07-4893-9bb4-bcda73c61b70 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 567.568277] env[63273]: WARNING nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Received unexpected event network-vif-plugged-ed28b0e4-1a07-4893-9bb4-bcda73c61b70 for instance with vm_state building and task_state spawning. [ 567.568277] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Received event network-changed-0ca9ef49-8de3-41cf-b25f-996926e56ea8 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 567.568277] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Refreshing instance network info cache due to event network-changed-0ca9ef49-8de3-41cf-b25f-996926e56ea8. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 567.568277] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Acquiring lock "refresh_cache-4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 567.568437] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Acquired lock "refresh_cache-4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 567.568437] env[63273]: DEBUG nova.network.neutron [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Refreshing network info cache for port 0ca9ef49-8de3-41cf-b25f-996926e56ea8 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 568.007520] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 568.008244] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 568.008673] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 568.451425] env[63273]: DEBUG nova.network.neutron [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Updating instance_info_cache with network_info: [{"id": "989e529e-348e-496a-8c6e-be4e8530306a", "address": "fa:16:3e:0f:82:68", "network": {"id": "31d84aea-5016-4a19-90f5-a0b8f7e1b4a4", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationTestJSON-439306553-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d806b4c5831b457fa075e387f55d8ad5", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a58387dd-f438-4913-af6a-fafb734cd881", "external-id": "nsx-vlan-transportzone-169", "segmentation_id": 169, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap989e529e-34", "ovs_interfaceid": "989e529e-348e-496a-8c6e-be4e8530306a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 568.475345] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Releasing lock "refresh_cache-a5c37718-0067-4f5e-ba7e-533f13529739" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 568.475730] env[63273]: DEBUG nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Instance network_info: |[{"id": "989e529e-348e-496a-8c6e-be4e8530306a", "address": "fa:16:3e:0f:82:68", "network": {"id": "31d84aea-5016-4a19-90f5-a0b8f7e1b4a4", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationTestJSON-439306553-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d806b4c5831b457fa075e387f55d8ad5", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a58387dd-f438-4913-af6a-fafb734cd881", "external-id": "nsx-vlan-transportzone-169", "segmentation_id": 169, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap989e529e-34", "ovs_interfaceid": "989e529e-348e-496a-8c6e-be4e8530306a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 568.477282] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:0f:82:68', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'a58387dd-f438-4913-af6a-fafb734cd881', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '989e529e-348e-496a-8c6e-be4e8530306a', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 568.490300] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Creating folder: Project (d806b4c5831b457fa075e387f55d8ad5). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 568.491105] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d000179b-9215-454b-be00-0ded40fe9586 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 568.503922] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Created folder: Project (d806b4c5831b457fa075e387f55d8ad5) in parent group-v986930. [ 568.504134] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Creating folder: Instances. Parent ref: group-v986958. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 568.508793] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-802acc21-820f-4373-9d81-e6b3d1243acb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 568.517890] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Created folder: Instances in parent group-v986958. [ 568.518169] env[63273]: DEBUG oslo.service.loopingcall [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 568.518369] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 568.518581] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-2d20a84e-a8c3-4907-8095-b5b0cc158a87 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 568.544207] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 568.544207] env[63273]: value = "task-5072004" [ 568.544207] env[63273]: _type = "Task" [ 568.544207] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 568.556759] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072004, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 568.903101] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquiring lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 568.903443] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 569.056728] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072004, 'name': CreateVM_Task} progress is 25%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 569.249682] env[63273]: DEBUG nova.network.neutron [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Updated VIF entry in instance network info cache for port 0ca9ef49-8de3-41cf-b25f-996926e56ea8. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 569.250166] env[63273]: DEBUG nova.network.neutron [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Updating instance_info_cache with network_info: [{"id": "0ca9ef49-8de3-41cf-b25f-996926e56ea8", "address": "fa:16:3e:0f:40:a6", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.173", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap0ca9ef49-8d", "ovs_interfaceid": "0ca9ef49-8de3-41cf-b25f-996926e56ea8", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 569.269353] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Releasing lock "refresh_cache-4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 569.269662] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Received event network-changed-ed28b0e4-1a07-4893-9bb4-bcda73c61b70 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 569.270591] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Refreshing instance network info cache due to event network-changed-ed28b0e4-1a07-4893-9bb4-bcda73c61b70. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 569.272353] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Acquiring lock "refresh_cache-02ed8dca-f6ae-47b0-a047-f89f858f5ff2" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 569.272566] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Acquired lock "refresh_cache-02ed8dca-f6ae-47b0-a047-f89f858f5ff2" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 569.273388] env[63273]: DEBUG nova.network.neutron [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Refreshing network info cache for port ed28b0e4-1a07-4893-9bb4-bcda73c61b70 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 569.569800] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072004, 'name': CreateVM_Task, 'duration_secs': 1.012663} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 569.570030] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 569.570837] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 569.570942] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 569.571282] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 569.571541] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-ea13a0a5-cd9a-49ea-b74f-72a54488bcea {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 569.578298] env[63273]: DEBUG oslo_vmware.api [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Waiting for the task: (returnval){ [ 569.578298] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52cd7797-ef4f-7621-e6ba-48cfa413a508" [ 569.578298] env[63273]: _type = "Task" [ 569.578298] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 569.592441] env[63273]: DEBUG oslo_vmware.api [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52cd7797-ef4f-7621-e6ba-48cfa413a508, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 569.657831] env[63273]: DEBUG nova.compute.manager [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Received event network-changed-aaf2e5cf-5e39-426b-a2ce-a81327b9c403 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 569.659044] env[63273]: DEBUG nova.compute.manager [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Refreshing instance network info cache due to event network-changed-aaf2e5cf-5e39-426b-a2ce-a81327b9c403. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 569.659427] env[63273]: DEBUG oslo_concurrency.lockutils [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] Acquiring lock "refresh_cache-11a32984-5548-4db4-8788-9b221ba381df" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 569.659598] env[63273]: DEBUG oslo_concurrency.lockutils [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] Acquired lock "refresh_cache-11a32984-5548-4db4-8788-9b221ba381df" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 569.660563] env[63273]: DEBUG nova.network.neutron [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Refreshing network info cache for port aaf2e5cf-5e39-426b-a2ce-a81327b9c403 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 569.929664] env[63273]: DEBUG nova.network.neutron [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Successfully updated port: e7c7fd55-2386-484e-b08a-2ce6f4da39fc {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 569.935603] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquiring lock "4ac5c733-988f-4428-ad6f-134d9f174e45" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 569.936821] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Lock "4ac5c733-988f-4428-ad6f-134d9f174e45" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 569.972010] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquiring lock "refresh_cache-94d19d7f-8eb7-4c93-aff4-3b1404338240" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 569.972010] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquired lock "refresh_cache-94d19d7f-8eb7-4c93-aff4-3b1404338240" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 569.972010] env[63273]: DEBUG nova.network.neutron [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 570.095148] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 570.095484] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 570.095878] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 570.401298] env[63273]: DEBUG nova.network.neutron [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 570.646058] env[63273]: DEBUG nova.network.neutron [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Updated VIF entry in instance network info cache for port ed28b0e4-1a07-4893-9bb4-bcda73c61b70. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 570.646916] env[63273]: DEBUG nova.network.neutron [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Updating instance_info_cache with network_info: [{"id": "ed28b0e4-1a07-4893-9bb4-bcda73c61b70", "address": "fa:16:3e:32:98:f3", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.180", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "taped28b0e4-1a", "ovs_interfaceid": "ed28b0e4-1a07-4893-9bb4-bcda73c61b70", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 570.657428] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Releasing lock "refresh_cache-02ed8dca-f6ae-47b0-a047-f89f858f5ff2" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 570.657688] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Received event network-vif-plugged-c479ac7d-7ab3-465a-8c20-ed211ff15840 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 570.657970] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Acquiring lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 570.658224] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 570.658371] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 570.658559] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] No waiting events found dispatching network-vif-plugged-c479ac7d-7ab3-465a-8c20-ed211ff15840 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 570.658702] env[63273]: WARNING nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Received unexpected event network-vif-plugged-c479ac7d-7ab3-465a-8c20-ed211ff15840 for instance with vm_state building and task_state spawning. [ 570.658875] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Received event network-changed-c479ac7d-7ab3-465a-8c20-ed211ff15840 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 570.659026] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Refreshing instance network info cache due to event network-changed-c479ac7d-7ab3-465a-8c20-ed211ff15840. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 570.659211] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Acquiring lock "refresh_cache-f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 570.659380] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Acquired lock "refresh_cache-f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 570.659547] env[63273]: DEBUG nova.network.neutron [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Refreshing network info cache for port c479ac7d-7ab3-465a-8c20-ed211ff15840 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 571.030032] env[63273]: DEBUG nova.compute.manager [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Received event network-changed-25e8da26-340c-419f-8abe-a9e907a9f001 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 571.030032] env[63273]: DEBUG nova.compute.manager [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Refreshing instance network info cache due to event network-changed-25e8da26-340c-419f-8abe-a9e907a9f001. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 571.030374] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Acquiring lock "refresh_cache-89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 571.030544] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Acquired lock "refresh_cache-89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 571.030742] env[63273]: DEBUG nova.network.neutron [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Refreshing network info cache for port 25e8da26-340c-419f-8abe-a9e907a9f001 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 571.306434] env[63273]: DEBUG nova.network.neutron [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Updating instance_info_cache with network_info: [{"id": "e7c7fd55-2386-484e-b08a-2ce6f4da39fc", "address": "fa:16:3e:d2:27:58", "network": {"id": "c652b419-7810-4332-96c9-8366a864b687", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationNegativeTestJSON-443175318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "4b844f0a784d4f20a5adbf7354c75cc7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "57691231-2b8d-4d71-8f79-d4a6a1d95ec8", "external-id": "nsx-vlan-transportzone-373", "segmentation_id": 373, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape7c7fd55-23", "ovs_interfaceid": "e7c7fd55-2386-484e-b08a-2ce6f4da39fc", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 571.322405] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Releasing lock "refresh_cache-94d19d7f-8eb7-4c93-aff4-3b1404338240" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 571.322731] env[63273]: DEBUG nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Instance network_info: |[{"id": "e7c7fd55-2386-484e-b08a-2ce6f4da39fc", "address": "fa:16:3e:d2:27:58", "network": {"id": "c652b419-7810-4332-96c9-8366a864b687", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationNegativeTestJSON-443175318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "4b844f0a784d4f20a5adbf7354c75cc7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "57691231-2b8d-4d71-8f79-d4a6a1d95ec8", "external-id": "nsx-vlan-transportzone-373", "segmentation_id": 373, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape7c7fd55-23", "ovs_interfaceid": "e7c7fd55-2386-484e-b08a-2ce6f4da39fc", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 571.323195] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:d2:27:58', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '57691231-2b8d-4d71-8f79-d4a6a1d95ec8', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'e7c7fd55-2386-484e-b08a-2ce6f4da39fc', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 571.333728] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Creating folder: Project (4b844f0a784d4f20a5adbf7354c75cc7). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 571.335144] env[63273]: DEBUG nova.network.neutron [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Updated VIF entry in instance network info cache for port aaf2e5cf-5e39-426b-a2ce-a81327b9c403. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 571.335487] env[63273]: DEBUG nova.network.neutron [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Updating instance_info_cache with network_info: [{"id": "aaf2e5cf-5e39-426b-a2ce-a81327b9c403", "address": "fa:16:3e:80:fa:32", "network": {"id": "c29ef6fe-282c-4f3f-b816-280b529e8985", "bridge": "br-int", "label": "tempest-ImagesOneServerNegativeTestJSON-477898296-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "93e7b65b07c44f5c88154e6f398bdbd8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "21310d90-efbc-45a8-a97f-c4358606530f", "external-id": "nsx-vlan-transportzone-672", "segmentation_id": 672, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapaaf2e5cf-5e", "ovs_interfaceid": "aaf2e5cf-5e39-426b-a2ce-a81327b9c403", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 571.341062] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-f10ab794-18cb-4302-a32f-3da247f38f72 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 571.347293] env[63273]: DEBUG oslo_concurrency.lockutils [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] Releasing lock "refresh_cache-11a32984-5548-4db4-8788-9b221ba381df" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 571.347613] env[63273]: DEBUG nova.compute.manager [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Received event network-vif-plugged-65bfc0ed-f2b0-48d3-a0cf-025b076c69ce {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 571.347810] env[63273]: DEBUG oslo_concurrency.lockutils [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] Acquiring lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 571.348031] env[63273]: DEBUG oslo_concurrency.lockutils [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] Lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 571.348296] env[63273]: DEBUG oslo_concurrency.lockutils [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] Lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 571.348467] env[63273]: DEBUG nova.compute.manager [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] No waiting events found dispatching network-vif-plugged-65bfc0ed-f2b0-48d3-a0cf-025b076c69ce {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 571.348670] env[63273]: WARNING nova.compute.manager [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Received unexpected event network-vif-plugged-65bfc0ed-f2b0-48d3-a0cf-025b076c69ce for instance with vm_state building and task_state spawning. [ 571.349075] env[63273]: DEBUG nova.compute.manager [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Received event network-changed-65bfc0ed-f2b0-48d3-a0cf-025b076c69ce {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 571.349075] env[63273]: DEBUG nova.compute.manager [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Refreshing instance network info cache due to event network-changed-65bfc0ed-f2b0-48d3-a0cf-025b076c69ce. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 571.349284] env[63273]: DEBUG oslo_concurrency.lockutils [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] Acquiring lock "refresh_cache-ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 571.349408] env[63273]: DEBUG oslo_concurrency.lockutils [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] Acquired lock "refresh_cache-ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 571.349870] env[63273]: DEBUG nova.network.neutron [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Refreshing network info cache for port 65bfc0ed-f2b0-48d3-a0cf-025b076c69ce {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 571.358561] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Created folder: Project (4b844f0a784d4f20a5adbf7354c75cc7) in parent group-v986930. [ 571.358561] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Creating folder: Instances. Parent ref: group-v986961. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 571.358561] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-28906201-c872-4ec3-a4ed-0af183057da0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 571.371303] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Created folder: Instances in parent group-v986961. [ 571.371799] env[63273]: DEBUG oslo.service.loopingcall [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 571.371799] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 571.372018] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-ce39fab3-6b75-49ac-b6ab-251402257803 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 571.396872] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 571.396872] env[63273]: value = "task-5072008" [ 571.396872] env[63273]: _type = "Task" [ 571.396872] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 571.406345] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072008, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 571.505160] env[63273]: DEBUG nova.network.neutron [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Updated VIF entry in instance network info cache for port 25e8da26-340c-419f-8abe-a9e907a9f001. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 571.505533] env[63273]: DEBUG nova.network.neutron [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Updating instance_info_cache with network_info: [{"id": "25e8da26-340c-419f-8abe-a9e907a9f001", "address": "fa:16:3e:a7:e0:c6", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.183", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap25e8da26-34", "ovs_interfaceid": "25e8da26-340c-419f-8abe-a9e907a9f001", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 571.521239] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Releasing lock "refresh_cache-89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 571.521501] env[63273]: DEBUG nova.compute.manager [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Received event network-vif-plugged-989e529e-348e-496a-8c6e-be4e8530306a {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 571.521695] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Acquiring lock "a5c37718-0067-4f5e-ba7e-533f13529739-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 571.521928] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Lock "a5c37718-0067-4f5e-ba7e-533f13529739-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 571.522113] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Lock "a5c37718-0067-4f5e-ba7e-533f13529739-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 571.522286] env[63273]: DEBUG nova.compute.manager [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] No waiting events found dispatching network-vif-plugged-989e529e-348e-496a-8c6e-be4e8530306a {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 571.522453] env[63273]: WARNING nova.compute.manager [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Received unexpected event network-vif-plugged-989e529e-348e-496a-8c6e-be4e8530306a for instance with vm_state building and task_state spawning. [ 571.522957] env[63273]: DEBUG nova.compute.manager [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Received event network-changed-989e529e-348e-496a-8c6e-be4e8530306a {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 571.522957] env[63273]: DEBUG nova.compute.manager [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Refreshing instance network info cache due to event network-changed-989e529e-348e-496a-8c6e-be4e8530306a. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 571.522957] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Acquiring lock "refresh_cache-a5c37718-0067-4f5e-ba7e-533f13529739" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 571.523943] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Acquired lock "refresh_cache-a5c37718-0067-4f5e-ba7e-533f13529739" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 571.524191] env[63273]: DEBUG nova.network.neutron [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Refreshing network info cache for port 989e529e-348e-496a-8c6e-be4e8530306a {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 571.904863] env[63273]: DEBUG nova.network.neutron [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Updated VIF entry in instance network info cache for port 989e529e-348e-496a-8c6e-be4e8530306a. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 571.905236] env[63273]: DEBUG nova.network.neutron [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Updating instance_info_cache with network_info: [{"id": "989e529e-348e-496a-8c6e-be4e8530306a", "address": "fa:16:3e:0f:82:68", "network": {"id": "31d84aea-5016-4a19-90f5-a0b8f7e1b4a4", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationTestJSON-439306553-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d806b4c5831b457fa075e387f55d8ad5", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a58387dd-f438-4913-af6a-fafb734cd881", "external-id": "nsx-vlan-transportzone-169", "segmentation_id": 169, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap989e529e-34", "ovs_interfaceid": "989e529e-348e-496a-8c6e-be4e8530306a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 571.913883] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072008, 'name': CreateVM_Task} progress is 99%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 571.924385] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Releasing lock "refresh_cache-a5c37718-0067-4f5e-ba7e-533f13529739" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 571.924748] env[63273]: DEBUG nova.compute.manager [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Received event network-vif-plugged-e7c7fd55-2386-484e-b08a-2ce6f4da39fc {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 571.924917] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Acquiring lock "94d19d7f-8eb7-4c93-aff4-3b1404338240-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 571.925182] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Lock "94d19d7f-8eb7-4c93-aff4-3b1404338240-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 571.925361] env[63273]: DEBUG oslo_concurrency.lockutils [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] Lock "94d19d7f-8eb7-4c93-aff4-3b1404338240-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 571.925527] env[63273]: DEBUG nova.compute.manager [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] No waiting events found dispatching network-vif-plugged-e7c7fd55-2386-484e-b08a-2ce6f4da39fc {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 571.925693] env[63273]: WARNING nova.compute.manager [req-01ce37c2-9142-4155-b816-eb751505a4eb req-42c2e784-842d-4d65-b2f3-f759204edb70 service nova] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Received unexpected event network-vif-plugged-e7c7fd55-2386-484e-b08a-2ce6f4da39fc for instance with vm_state building and task_state spawning. [ 571.973513] env[63273]: DEBUG nova.network.neutron [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Updated VIF entry in instance network info cache for port c479ac7d-7ab3-465a-8c20-ed211ff15840. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 571.973513] env[63273]: DEBUG nova.network.neutron [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Updating instance_info_cache with network_info: [{"id": "c479ac7d-7ab3-465a-8c20-ed211ff15840", "address": "fa:16:3e:b1:90:48", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.65", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc479ac7d-7a", "ovs_interfaceid": "c479ac7d-7ab3-465a-8c20-ed211ff15840", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 571.990797] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Releasing lock "refresh_cache-f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 571.991071] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Received event network-vif-plugged-25e8da26-340c-419f-8abe-a9e907a9f001 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 571.991267] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Acquiring lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 571.991475] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 571.991644] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] Lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 571.991813] env[63273]: DEBUG nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] No waiting events found dispatching network-vif-plugged-25e8da26-340c-419f-8abe-a9e907a9f001 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 571.991979] env[63273]: WARNING nova.compute.manager [req-c99f3fa6-af9b-4c10-9077-721d7580816d req-78b3dcf0-6b31-44d2-b533-f374b22eff37 service nova] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Received unexpected event network-vif-plugged-25e8da26-340c-419f-8abe-a9e907a9f001 for instance with vm_state building and task_state spawning. [ 572.380161] env[63273]: DEBUG nova.network.neutron [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Updated VIF entry in instance network info cache for port 65bfc0ed-f2b0-48d3-a0cf-025b076c69ce. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 572.380161] env[63273]: DEBUG nova.network.neutron [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Updating instance_info_cache with network_info: [{"id": "65bfc0ed-f2b0-48d3-a0cf-025b076c69ce", "address": "fa:16:3e:5a:d3:50", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap65bfc0ed-f2", "ovs_interfaceid": "65bfc0ed-f2b0-48d3-a0cf-025b076c69ce", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 572.393974] env[63273]: DEBUG oslo_concurrency.lockutils [req-de2c9f9e-7579-4794-87ee-dd9d133a139a req-1026b7f6-f21d-4e12-9d00-61c6c8430534 service nova] Releasing lock "refresh_cache-ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 572.409397] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072008, 'name': CreateVM_Task, 'duration_secs': 0.598394} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 572.409397] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 572.409988] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 572.410193] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 572.411120] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 572.411120] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-002b7593-c558-4500-ad5d-8371a74a103e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 572.417762] env[63273]: DEBUG oslo_vmware.api [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Waiting for the task: (returnval){ [ 572.417762] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52f5c7ef-ee5a-0b32-4ccc-f8601f39e4ea" [ 572.417762] env[63273]: _type = "Task" [ 572.417762] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 572.427257] env[63273]: DEBUG oslo_vmware.api [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52f5c7ef-ee5a-0b32-4ccc-f8601f39e4ea, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 572.937862] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 572.938158] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 572.938429] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 573.940462] env[63273]: DEBUG nova.compute.manager [req-29436212-2728-4f4e-9a0e-3a8866801c29 req-524e5ddb-0310-4f1c-9fbe-be1fece9ffbd service nova] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Received event network-changed-e7c7fd55-2386-484e-b08a-2ce6f4da39fc {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 573.940462] env[63273]: DEBUG nova.compute.manager [req-29436212-2728-4f4e-9a0e-3a8866801c29 req-524e5ddb-0310-4f1c-9fbe-be1fece9ffbd service nova] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Refreshing instance network info cache due to event network-changed-e7c7fd55-2386-484e-b08a-2ce6f4da39fc. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 573.940462] env[63273]: DEBUG oslo_concurrency.lockutils [req-29436212-2728-4f4e-9a0e-3a8866801c29 req-524e5ddb-0310-4f1c-9fbe-be1fece9ffbd service nova] Acquiring lock "refresh_cache-94d19d7f-8eb7-4c93-aff4-3b1404338240" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 573.940462] env[63273]: DEBUG oslo_concurrency.lockutils [req-29436212-2728-4f4e-9a0e-3a8866801c29 req-524e5ddb-0310-4f1c-9fbe-be1fece9ffbd service nova] Acquired lock "refresh_cache-94d19d7f-8eb7-4c93-aff4-3b1404338240" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 573.940462] env[63273]: DEBUG nova.network.neutron [req-29436212-2728-4f4e-9a0e-3a8866801c29 req-524e5ddb-0310-4f1c-9fbe-be1fece9ffbd service nova] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Refreshing network info cache for port e7c7fd55-2386-484e-b08a-2ce6f4da39fc {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 574.726718] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8cd7758d-9e5f-463b-9166-d44861ef7b72 tempest-AttachInterfacesV270Test-389945933 tempest-AttachInterfacesV270Test-389945933-project-member] Acquiring lock "3af8fe9a-643b-4650-a036-6f1b9c3c8d03" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 574.727186] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8cd7758d-9e5f-463b-9166-d44861ef7b72 tempest-AttachInterfacesV270Test-389945933 tempest-AttachInterfacesV270Test-389945933-project-member] Lock "3af8fe9a-643b-4650-a036-6f1b9c3c8d03" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.067713] env[63273]: DEBUG nova.network.neutron [req-29436212-2728-4f4e-9a0e-3a8866801c29 req-524e5ddb-0310-4f1c-9fbe-be1fece9ffbd service nova] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Updated VIF entry in instance network info cache for port e7c7fd55-2386-484e-b08a-2ce6f4da39fc. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 575.069630] env[63273]: DEBUG nova.network.neutron [req-29436212-2728-4f4e-9a0e-3a8866801c29 req-524e5ddb-0310-4f1c-9fbe-be1fece9ffbd service nova] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Updating instance_info_cache with network_info: [{"id": "e7c7fd55-2386-484e-b08a-2ce6f4da39fc", "address": "fa:16:3e:d2:27:58", "network": {"id": "c652b419-7810-4332-96c9-8366a864b687", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationNegativeTestJSON-443175318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "4b844f0a784d4f20a5adbf7354c75cc7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "57691231-2b8d-4d71-8f79-d4a6a1d95ec8", "external-id": "nsx-vlan-transportzone-373", "segmentation_id": 373, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape7c7fd55-23", "ovs_interfaceid": "e7c7fd55-2386-484e-b08a-2ce6f4da39fc", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 575.081337] env[63273]: DEBUG oslo_concurrency.lockutils [req-29436212-2728-4f4e-9a0e-3a8866801c29 req-524e5ddb-0310-4f1c-9fbe-be1fece9ffbd service nova] Releasing lock "refresh_cache-94d19d7f-8eb7-4c93-aff4-3b1404338240" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 577.839667] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b42cefbb-26e0-40af-9324-00e8a9e4b861 tempest-TenantUsagesTestJSON-1769228805 tempest-TenantUsagesTestJSON-1769228805-project-member] Acquiring lock "7d000257-5fc1-43fa-9fc5-a07519a6feea" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 577.840065] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b42cefbb-26e0-40af-9324-00e8a9e4b861 tempest-TenantUsagesTestJSON-1769228805 tempest-TenantUsagesTestJSON-1769228805-project-member] Lock "7d000257-5fc1-43fa-9fc5-a07519a6feea" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 578.972103] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7c3e6778-2ab0-4c2b-9b3a-29c19c09b3bb tempest-ListImageFiltersTestJSON-45654626 tempest-ListImageFiltersTestJSON-45654626-project-member] Acquiring lock "68fe9a49-e561-45da-afcb-e804e2109e76" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 578.972453] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7c3e6778-2ab0-4c2b-9b3a-29c19c09b3bb tempest-ListImageFiltersTestJSON-45654626 tempest-ListImageFiltersTestJSON-45654626-project-member] Lock "68fe9a49-e561-45da-afcb-e804e2109e76" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 579.405393] env[63273]: DEBUG oslo_concurrency.lockutils [None req-e89c7380-dba5-40be-bfcb-bc0b02bea0fa tempest-ServersWithSpecificFlavorTestJSON-1103463149 tempest-ServersWithSpecificFlavorTestJSON-1103463149-project-member] Acquiring lock "8dd5ad91-d09a-41dd-a8d0-fd22e96fc279" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 579.405609] env[63273]: DEBUG oslo_concurrency.lockutils [None req-e89c7380-dba5-40be-bfcb-bc0b02bea0fa tempest-ServersWithSpecificFlavorTestJSON-1103463149 tempest-ServersWithSpecificFlavorTestJSON-1103463149-project-member] Lock "8dd5ad91-d09a-41dd-a8d0-fd22e96fc279" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 580.434571] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2d4f2721-535d-4a81-a48b-76c005a769e7 tempest-ListImageFiltersTestJSON-45654626 tempest-ListImageFiltersTestJSON-45654626-project-member] Acquiring lock "8728833b-c3ac-49b2-9a33-ecfae9597e9e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 580.434923] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2d4f2721-535d-4a81-a48b-76c005a769e7 tempest-ListImageFiltersTestJSON-45654626 tempest-ListImageFiltersTestJSON-45654626-project-member] Lock "8728833b-c3ac-49b2-9a33-ecfae9597e9e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 583.033602] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0ec976f8-3e01-4399-a146-318088183878 tempest-VolumesAssistedSnapshotsTest-905484168 tempest-VolumesAssistedSnapshotsTest-905484168-project-member] Acquiring lock "6699f6e7-203d-4d8a-bc9e-8c8d5427db36" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 583.033913] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0ec976f8-3e01-4399-a146-318088183878 tempest-VolumesAssistedSnapshotsTest-905484168 tempest-VolumesAssistedSnapshotsTest-905484168-project-member] Lock "6699f6e7-203d-4d8a-bc9e-8c8d5427db36" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 584.966343] env[63273]: DEBUG oslo_concurrency.lockutils [None req-27626105-34c4-40c9-8bcd-2c38276131a3 tempest-ServersTestBootFromVolume-777103340 tempest-ServersTestBootFromVolume-777103340-project-member] Acquiring lock "362b7115-a270-407f-a53c-506c863c8fe1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 584.966696] env[63273]: DEBUG oslo_concurrency.lockutils [None req-27626105-34c4-40c9-8bcd-2c38276131a3 tempest-ServersTestBootFromVolume-777103340 tempest-ServersTestBootFromVolume-777103340-project-member] Lock "362b7115-a270-407f-a53c-506c863c8fe1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 587.564574] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c402fea3-ce13-4ff0-864f-0f1bdb3edb6e tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] Acquiring lock "cc6f5d6c-1f3c-4ead-8ee4-9b59923ac57b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 587.564945] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c402fea3-ce13-4ff0-864f-0f1bdb3edb6e tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] Lock "cc6f5d6c-1f3c-4ead-8ee4-9b59923ac57b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.014931] env[63273]: DEBUG oslo_concurrency.lockutils [None req-26ede9af-6526-4b79-b9b1-5be31b9f9dac tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] Acquiring lock "811bed85-6a4d-4340-9e4f-a2acd86d921c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 589.016376] env[63273]: DEBUG oslo_concurrency.lockutils [None req-26ede9af-6526-4b79-b9b1-5be31b9f9dac tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] Lock "811bed85-6a4d-4340-9e4f-a2acd86d921c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.002s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.880671] env[63273]: DEBUG oslo_concurrency.lockutils [None req-aa94caae-be63-44fb-aa6d-314d5d081b13 tempest-ServerShowV247Test-1154040461 tempest-ServerShowV247Test-1154040461-project-member] Acquiring lock "707e02eb-1471-4ddd-be9e-a41f33d42482" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 589.880910] env[63273]: DEBUG oslo_concurrency.lockutils [None req-aa94caae-be63-44fb-aa6d-314d5d081b13 tempest-ServerShowV247Test-1154040461 tempest-ServerShowV247Test-1154040461-project-member] Lock "707e02eb-1471-4ddd-be9e-a41f33d42482" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 590.442693] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ce1ab873-88d5-48ef-ba5e-2f411f0a5350 tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] Acquiring lock "4846a37e-b175-4f23-b8c1-34a6b124513d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 590.442975] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ce1ab873-88d5-48ef-ba5e-2f411f0a5350 tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] Lock "4846a37e-b175-4f23-b8c1-34a6b124513d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 592.879379] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6c42ddf1-295d-4de0-b309-d4e3c257cc87 tempest-ServerShowV247Test-1154040461 tempest-ServerShowV247Test-1154040461-project-member] Acquiring lock "11838596-1730-4c23-bbf2-3159d3c6d72d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 592.879892] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6c42ddf1-295d-4de0-b309-d4e3c257cc87 tempest-ServerShowV247Test-1154040461 tempest-ServerShowV247Test-1154040461-project-member] Lock "11838596-1730-4c23-bbf2-3159d3c6d72d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 592.883427] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25fa772e-cd3c-434f-b968-53f657928c1b tempest-ServerExternalEventsTest-1759374695 tempest-ServerExternalEventsTest-1759374695-project-member] Acquiring lock "717fd312-ac1d-42b6-9f93-13b9f235bd23" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 592.883636] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25fa772e-cd3c-434f-b968-53f657928c1b tempest-ServerExternalEventsTest-1759374695 tempest-ServerExternalEventsTest-1759374695-project-member] Lock "717fd312-ac1d-42b6-9f93-13b9f235bd23" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 597.309825] env[63273]: DEBUG oslo_concurrency.lockutils [None req-530971e6-1c95-4e35-8faa-c89e61642e17 tempest-ServerMetadataNegativeTestJSON-1866644747 tempest-ServerMetadataNegativeTestJSON-1866644747-project-member] Acquiring lock "a12ff1bf-8cfa-4d03-828d-6b34727f61bf" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 597.310194] env[63273]: DEBUG oslo_concurrency.lockutils [None req-530971e6-1c95-4e35-8faa-c89e61642e17 tempest-ServerMetadataNegativeTestJSON-1866644747 tempest-ServerMetadataNegativeTestJSON-1866644747-project-member] Lock "a12ff1bf-8cfa-4d03-828d-6b34727f61bf" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 598.416254] env[63273]: DEBUG oslo_concurrency.lockutils [None req-294aff5a-b2ea-40b8-885c-f8bde42d2b30 tempest-InstanceActionsNegativeTestJSON-4733633 tempest-InstanceActionsNegativeTestJSON-4733633-project-member] Acquiring lock "f889087d-5f22-4de6-8dff-819d2afb7240" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 598.416688] env[63273]: DEBUG oslo_concurrency.lockutils [None req-294aff5a-b2ea-40b8-885c-f8bde42d2b30 tempest-InstanceActionsNegativeTestJSON-4733633 tempest-InstanceActionsNegativeTestJSON-4733633-project-member] Lock "f889087d-5f22-4de6-8dff-819d2afb7240" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 600.103999] env[63273]: WARNING oslo_vmware.rw_handles [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 600.103999] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 600.103999] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 600.103999] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 600.103999] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 600.103999] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 600.103999] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 600.103999] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 600.103999] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 600.103999] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 600.103999] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 600.103999] env[63273]: ERROR oslo_vmware.rw_handles [ 600.104830] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/9ee33103-217c-4e6c-8094-d3900a146f5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 600.106371] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 600.106756] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Copying Virtual Disk [datastore1] vmware_temp/9ee33103-217c-4e6c-8094-d3900a146f5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/9ee33103-217c-4e6c-8094-d3900a146f5a/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 600.106998] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-eeeb7468-9770-46a6-ad5f-c314f84dc3de {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.119250] env[63273]: DEBUG oslo_vmware.api [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Waiting for the task: (returnval){ [ 600.119250] env[63273]: value = "task-5072014" [ 600.119250] env[63273]: _type = "Task" [ 600.119250] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 600.129615] env[63273]: DEBUG oslo_vmware.api [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Task: {'id': task-5072014, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 600.416736] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9745a7f0-5825-4992-90f1-1e5f105cf947 tempest-ServerActionsTestJSON-2016287253 tempest-ServerActionsTestJSON-2016287253-project-member] Acquiring lock "a4e56801-a8b8-448f-9b1a-d228ceeaaa46" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 600.416961] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9745a7f0-5825-4992-90f1-1e5f105cf947 tempest-ServerActionsTestJSON-2016287253 tempest-ServerActionsTestJSON-2016287253-project-member] Lock "a4e56801-a8b8-448f-9b1a-d228ceeaaa46" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 600.631068] env[63273]: DEBUG oslo_vmware.exceptions [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 600.631306] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 600.634868] env[63273]: ERROR nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 600.634868] env[63273]: Faults: ['InvalidArgument'] [ 600.634868] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Traceback (most recent call last): [ 600.634868] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 600.634868] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] yield resources [ 600.634868] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 600.634868] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] self.driver.spawn(context, instance, image_meta, [ 600.634868] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 600.634868] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 600.634868] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 600.634868] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] self._fetch_image_if_missing(context, vi) [ 600.634868] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] image_cache(vi, tmp_image_ds_loc) [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] vm_util.copy_virtual_disk( [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] session._wait_for_task(vmdk_copy_task) [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] return self.wait_for_task(task_ref) [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] return evt.wait() [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] result = hub.switch() [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 600.635342] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] return self.greenlet.switch() [ 600.635817] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 600.635817] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] self.f(*self.args, **self.kw) [ 600.635817] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 600.635817] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] raise exceptions.translate_fault(task_info.error) [ 600.635817] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 600.635817] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Faults: ['InvalidArgument'] [ 600.635817] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] [ 600.635817] env[63273]: INFO nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Terminating instance [ 600.637560] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 600.637560] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 600.638216] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquiring lock "refresh_cache-f948a055-fe10-4b55-b2e7-64f544c2d11a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 600.638216] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquired lock "refresh_cache-f948a055-fe10-4b55-b2e7-64f544c2d11a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 600.638216] env[63273]: DEBUG nova.network.neutron [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 600.638948] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-d68646af-77f1-44c0-bbf5-3e8cf77e01c9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.652910] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 600.653205] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 600.654247] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-e6a9cedc-b81d-4900-83dc-0aa191b63c6c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.661704] env[63273]: DEBUG oslo_vmware.api [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Waiting for the task: (returnval){ [ 600.661704] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e2fcb1-3339-0c02-77cd-fe64d6a414d3" [ 600.661704] env[63273]: _type = "Task" [ 600.661704] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 600.671839] env[63273]: DEBUG oslo_vmware.api [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e2fcb1-3339-0c02-77cd-fe64d6a414d3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 600.684968] env[63273]: DEBUG nova.network.neutron [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 600.841859] env[63273]: DEBUG nova.network.neutron [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 600.853693] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Releasing lock "refresh_cache-f948a055-fe10-4b55-b2e7-64f544c2d11a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 600.854130] env[63273]: DEBUG nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 600.854325] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 600.855550] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a8d858d-ece1-4674-bbc6-f909faf68065 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.869819] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 600.870121] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-b3633679-d28b-452c-a3f9-33e64e6cedd8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.902742] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 600.903073] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 600.903294] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Deleting the datastore file [datastore1] f948a055-fe10-4b55-b2e7-64f544c2d11a {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 600.903581] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-64195403-a686-4ecb-931b-f3205827dcd9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.913386] env[63273]: DEBUG oslo_vmware.api [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Waiting for the task: (returnval){ [ 600.913386] env[63273]: value = "task-5072016" [ 600.913386] env[63273]: _type = "Task" [ 600.913386] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 600.922945] env[63273]: DEBUG oslo_vmware.api [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Task: {'id': task-5072016, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 601.173640] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 601.173954] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Creating directory with path [datastore1] vmware_temp/b32b776b-fcca-4355-8b63-5164e7798599/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 601.174164] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0b104314-7c07-4b55-83d8-3761bec70566 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.188558] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Created directory with path [datastore1] vmware_temp/b32b776b-fcca-4355-8b63-5164e7798599/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 601.188999] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Fetch image to [datastore1] vmware_temp/b32b776b-fcca-4355-8b63-5164e7798599/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 601.188999] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/b32b776b-fcca-4355-8b63-5164e7798599/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 601.189764] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b68c006-69ab-46ca-a734-4810cbc34b28 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.198862] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc22ffa9-c08f-4543-b208-c9493c1db706 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.211881] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f56dfde1-6841-44fd-a870-b7676f578520 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.247851] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f84b43b8-81ca-4df4-a6c1-c98443e2d1e7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.257394] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-9e9b70cb-d9ef-49c5-be96-efe9d6743423 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.354953] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 601.426509] env[63273]: DEBUG oslo_vmware.api [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Task: {'id': task-5072016, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.037239} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 601.426628] env[63273]: DEBUG oslo_vmware.rw_handles [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/b32b776b-fcca-4355-8b63-5164e7798599/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 601.429194] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 601.429790] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 601.430219] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 601.430933] env[63273]: INFO nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Took 0.58 seconds to destroy the instance on the hypervisor. [ 601.431590] env[63273]: DEBUG oslo.service.loopingcall [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 601.432826] env[63273]: DEBUG nova.compute.manager [-] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Skipping network deallocation for instance since networking was not requested. {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2277}} [ 601.490377] env[63273]: DEBUG nova.compute.claims [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 601.490637] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.490910] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 601.496402] env[63273]: DEBUG oslo_vmware.rw_handles [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 601.496584] env[63273]: DEBUG oslo_vmware.rw_handles [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/b32b776b-fcca-4355-8b63-5164e7798599/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 602.053792] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3eba476f-6d2a-4e2b-b77a-e72d403675f1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.066022] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea9efeff-5c46-4963-a40e-8ee8ebf22ccb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.105264] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87abff07-beb6-4b3e-b345-c96863893ec3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.114761] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2458b472-6917-47ab-bad2-6d6c0dee662b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.133714] env[63273]: DEBUG nova.compute.provider_tree [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 602.146815] env[63273]: DEBUG nova.scheduler.client.report [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 602.170440] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.679s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 602.171100] env[63273]: ERROR nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 602.171100] env[63273]: Faults: ['InvalidArgument'] [ 602.171100] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Traceback (most recent call last): [ 602.171100] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 602.171100] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] self.driver.spawn(context, instance, image_meta, [ 602.171100] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 602.171100] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 602.171100] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 602.171100] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] self._fetch_image_if_missing(context, vi) [ 602.171100] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 602.171100] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] image_cache(vi, tmp_image_ds_loc) [ 602.171100] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] vm_util.copy_virtual_disk( [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] session._wait_for_task(vmdk_copy_task) [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] return self.wait_for_task(task_ref) [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] return evt.wait() [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] result = hub.switch() [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] return self.greenlet.switch() [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 602.171606] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] self.f(*self.args, **self.kw) [ 602.172052] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 602.172052] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] raise exceptions.translate_fault(task_info.error) [ 602.172052] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 602.172052] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Faults: ['InvalidArgument'] [ 602.172052] env[63273]: ERROR nova.compute.manager [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] [ 602.172052] env[63273]: DEBUG nova.compute.utils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 602.178685] env[63273]: DEBUG nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Build of instance f948a055-fe10-4b55-b2e7-64f544c2d11a was re-scheduled: A specified parameter was not correct: fileType [ 602.178685] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 602.179093] env[63273]: DEBUG nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 602.179410] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquiring lock "refresh_cache-f948a055-fe10-4b55-b2e7-64f544c2d11a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 602.179581] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Acquired lock "refresh_cache-f948a055-fe10-4b55-b2e7-64f544c2d11a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 602.179750] env[63273]: DEBUG nova.network.neutron [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 602.213121] env[63273]: DEBUG nova.network.neutron [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 602.395949] env[63273]: DEBUG nova.network.neutron [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 602.409095] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Releasing lock "refresh_cache-f948a055-fe10-4b55-b2e7-64f544c2d11a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 602.409244] env[63273]: DEBUG nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 602.409434] env[63273]: DEBUG nova.compute.manager [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] Skipping network deallocation for instance since networking was not requested. {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2277}} [ 602.566078] env[63273]: INFO nova.scheduler.client.report [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Deleted allocations for instance f948a055-fe10-4b55-b2e7-64f544c2d11a [ 602.599780] env[63273]: DEBUG oslo_concurrency.lockutils [None req-cc41a4f4-00ef-44d7-a440-8f1f45fd1734 tempest-ServersAdmin275Test-1814094412 tempest-ServersAdmin275Test-1814094412-project-member] Lock "f948a055-fe10-4b55-b2e7-64f544c2d11a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 55.614s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 602.602448] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "f948a055-fe10-4b55-b2e7-64f544c2d11a" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 38.326s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 602.602448] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f948a055-fe10-4b55-b2e7-64f544c2d11a] During sync_power_state the instance has a pending task (spawning). Skip. [ 602.602448] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "f948a055-fe10-4b55-b2e7-64f544c2d11a" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 602.661133] env[63273]: DEBUG nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 602.742488] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 602.742760] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 602.744304] env[63273]: INFO nova.compute.claims [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 603.319335] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4dbd536-d5c7-48c5-96e4-19c53933356e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.330630] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98afde43-f6c8-4411-bfc3-655a95521891 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.365718] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b177b22-4701-4acb-905f-bed843ad5917 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.378073] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69e3451f-ce92-4710-9f1b-d1cf32eb125c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.393901] env[63273]: DEBUG nova.compute.provider_tree [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 603.408117] env[63273]: DEBUG nova.scheduler.client.report [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 603.431286] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.688s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 603.431286] env[63273]: DEBUG nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 603.489192] env[63273]: DEBUG nova.compute.utils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 603.490719] env[63273]: DEBUG nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 603.490985] env[63273]: DEBUG nova.network.neutron [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 603.504163] env[63273]: DEBUG nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 603.585348] env[63273]: DEBUG nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 603.617243] env[63273]: DEBUG nova.virt.hardware [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 603.617499] env[63273]: DEBUG nova.virt.hardware [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 603.617658] env[63273]: DEBUG nova.virt.hardware [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 603.617835] env[63273]: DEBUG nova.virt.hardware [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 603.618016] env[63273]: DEBUG nova.virt.hardware [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 603.618197] env[63273]: DEBUG nova.virt.hardware [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 603.618431] env[63273]: DEBUG nova.virt.hardware [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 603.618663] env[63273]: DEBUG nova.virt.hardware [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 603.618795] env[63273]: DEBUG nova.virt.hardware [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 603.618965] env[63273]: DEBUG nova.virt.hardware [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 603.619177] env[63273]: DEBUG nova.virt.hardware [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 603.620060] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b8425c0-2330-4cde-a9bb-7ab7845a2401 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.629668] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a50a5db-5599-4a92-ba1a-44077e8ff8c3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.722985] env[63273]: DEBUG nova.policy [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e607fb95c602421699a05b9541652799', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ad03061e544f4b17abb83d9596c461a0', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 605.338149] env[63273]: DEBUG nova.network.neutron [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Successfully created port: ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 607.420672] env[63273]: DEBUG nova.network.neutron [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Successfully updated port: ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 607.439427] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquiring lock "refresh_cache-5501c192-867b-40c0-9bea-c3c44865d2f4" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 607.440096] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquired lock "refresh_cache-5501c192-867b-40c0-9bea-c3c44865d2f4" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 607.440096] env[63273]: DEBUG nova.network.neutron [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 607.523772] env[63273]: DEBUG nova.network.neutron [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 607.845971] env[63273]: DEBUG nova.network.neutron [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Updating instance_info_cache with network_info: [{"id": "ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd", "address": "fa:16:3e:a0:af:80", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.116", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapec5a1ce5-c9", "ovs_interfaceid": "ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 607.861985] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Releasing lock "refresh_cache-5501c192-867b-40c0-9bea-c3c44865d2f4" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 607.862350] env[63273]: DEBUG nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Instance network_info: |[{"id": "ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd", "address": "fa:16:3e:a0:af:80", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.116", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapec5a1ce5-c9", "ovs_interfaceid": "ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 607.864913] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:a0:af:80', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f66f8375-4460-4acd-987b-acda72bfcf0d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 607.873067] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Creating folder: Project (ad03061e544f4b17abb83d9596c461a0). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 607.873067] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-641e8509-4290-4b65-b1db-41390630c883 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.883476] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Created folder: Project (ad03061e544f4b17abb83d9596c461a0) in parent group-v986930. [ 607.883543] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Creating folder: Instances. Parent ref: group-v986965. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 607.883944] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-6d0eb4d0-2d20-4393-bd28-3bc42ff917fe {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.894400] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Created folder: Instances in parent group-v986965. [ 607.894400] env[63273]: DEBUG oslo.service.loopingcall [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 607.894400] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 607.894400] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-330420e2-b03b-4796-850e-202c8ea603ca {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.915691] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 607.915691] env[63273]: value = "task-5072019" [ 607.915691] env[63273]: _type = "Task" [ 607.915691] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 607.924360] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072019, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 608.412524] env[63273]: DEBUG nova.compute.manager [req-5732af9c-de01-4fab-a073-c66556b9c42d req-82b5d152-c850-460f-8afb-af6b77fae8b0 service nova] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Received event network-vif-plugged-ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 608.412524] env[63273]: DEBUG oslo_concurrency.lockutils [req-5732af9c-de01-4fab-a073-c66556b9c42d req-82b5d152-c850-460f-8afb-af6b77fae8b0 service nova] Acquiring lock "5501c192-867b-40c0-9bea-c3c44865d2f4-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 608.412524] env[63273]: DEBUG oslo_concurrency.lockutils [req-5732af9c-de01-4fab-a073-c66556b9c42d req-82b5d152-c850-460f-8afb-af6b77fae8b0 service nova] Lock "5501c192-867b-40c0-9bea-c3c44865d2f4-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 608.412524] env[63273]: DEBUG oslo_concurrency.lockutils [req-5732af9c-de01-4fab-a073-c66556b9c42d req-82b5d152-c850-460f-8afb-af6b77fae8b0 service nova] Lock "5501c192-867b-40c0-9bea-c3c44865d2f4-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 608.412746] env[63273]: DEBUG nova.compute.manager [req-5732af9c-de01-4fab-a073-c66556b9c42d req-82b5d152-c850-460f-8afb-af6b77fae8b0 service nova] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] No waiting events found dispatching network-vif-plugged-ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 608.413056] env[63273]: WARNING nova.compute.manager [req-5732af9c-de01-4fab-a073-c66556b9c42d req-82b5d152-c850-460f-8afb-af6b77fae8b0 service nova] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Received unexpected event network-vif-plugged-ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd for instance with vm_state building and task_state spawning. [ 608.427101] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072019, 'name': CreateVM_Task, 'duration_secs': 0.362921} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 608.427101] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 608.428422] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 608.428422] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 608.428422] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 608.428422] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-719c9c76-674f-4762-acf1-91e32409b149 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.438517] env[63273]: DEBUG oslo_vmware.api [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Waiting for the task: (returnval){ [ 608.438517] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ce1a98-f146-03fb-8e4c-bdd0c06bfe3d" [ 608.438517] env[63273]: _type = "Task" [ 608.438517] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 608.449292] env[63273]: DEBUG oslo_vmware.api [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ce1a98-f146-03fb-8e4c-bdd0c06bfe3d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 608.956231] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 608.959163] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 608.959163] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 609.064978] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquiring lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 609.065496] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 609.100638] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquiring lock "1979e66a-172d-4130-90ea-9f0034b13071" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 609.101755] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "1979e66a-172d-4130-90ea-9f0034b13071" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 609.135498] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquiring lock "6b48c313-e3d5-4d3c-83c5-3c12d948abba" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 609.136379] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "6b48c313-e3d5-4d3c-83c5-3c12d948abba" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.002s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 611.341263] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9be77a4a-85b3-4628-bb42-cb12147dbd02 tempest-VolumesAdminNegativeTest-184811883 tempest-VolumesAdminNegativeTest-184811883-project-member] Acquiring lock "c936b733-7faa-4e73-99c5-2ac11d5421ff" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 611.341682] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9be77a4a-85b3-4628-bb42-cb12147dbd02 tempest-VolumesAdminNegativeTest-184811883 tempest-VolumesAdminNegativeTest-184811883-project-member] Lock "c936b733-7faa-4e73-99c5-2ac11d5421ff" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 611.403739] env[63273]: DEBUG nova.compute.manager [req-bbe9319a-d448-44f9-901b-2bf2789b2f28 req-0ac972e4-ef3d-4849-913c-8597a76c0e8f service nova] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Received event network-changed-ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 611.404598] env[63273]: DEBUG nova.compute.manager [req-bbe9319a-d448-44f9-901b-2bf2789b2f28 req-0ac972e4-ef3d-4849-913c-8597a76c0e8f service nova] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Refreshing instance network info cache due to event network-changed-ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 611.404598] env[63273]: DEBUG oslo_concurrency.lockutils [req-bbe9319a-d448-44f9-901b-2bf2789b2f28 req-0ac972e4-ef3d-4849-913c-8597a76c0e8f service nova] Acquiring lock "refresh_cache-5501c192-867b-40c0-9bea-c3c44865d2f4" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 611.404598] env[63273]: DEBUG oslo_concurrency.lockutils [req-bbe9319a-d448-44f9-901b-2bf2789b2f28 req-0ac972e4-ef3d-4849-913c-8597a76c0e8f service nova] Acquired lock "refresh_cache-5501c192-867b-40c0-9bea-c3c44865d2f4" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 611.404598] env[63273]: DEBUG nova.network.neutron [req-bbe9319a-d448-44f9-901b-2bf2789b2f28 req-0ac972e4-ef3d-4849-913c-8597a76c0e8f service nova] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Refreshing network info cache for port ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 612.168014] env[63273]: DEBUG nova.network.neutron [req-bbe9319a-d448-44f9-901b-2bf2789b2f28 req-0ac972e4-ef3d-4849-913c-8597a76c0e8f service nova] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Updated VIF entry in instance network info cache for port ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 612.173264] env[63273]: DEBUG nova.network.neutron [req-bbe9319a-d448-44f9-901b-2bf2789b2f28 req-0ac972e4-ef3d-4849-913c-8597a76c0e8f service nova] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Updating instance_info_cache with network_info: [{"id": "ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd", "address": "fa:16:3e:a0:af:80", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.116", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapec5a1ce5-c9", "ovs_interfaceid": "ec5a1ce5-c9db-4556-bdc7-73fe17acc7bd", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 612.187253] env[63273]: DEBUG oslo_concurrency.lockutils [req-bbe9319a-d448-44f9-901b-2bf2789b2f28 req-0ac972e4-ef3d-4849-913c-8597a76c0e8f service nova] Releasing lock "refresh_cache-5501c192-867b-40c0-9bea-c3c44865d2f4" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 615.267122] env[63273]: DEBUG oslo_concurrency.lockutils [None req-28203e26-125e-4639-bf38-b7aef42c13a5 tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] Acquiring lock "ff6cf180-c0fe-4b4f-9bac-08e9e3dc7c1d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 615.267591] env[63273]: DEBUG oslo_concurrency.lockutils [None req-28203e26-125e-4639-bf38-b7aef42c13a5 tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] Lock "ff6cf180-c0fe-4b4f-9bac-08e9e3dc7c1d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 621.809557] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 621.846814] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 621.846992] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 621.847133] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 621.871592] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 621.871695] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 621.871831] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 621.871971] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 621.872144] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 621.872352] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 621.872514] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 621.872667] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 621.872803] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 621.872937] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 621.873084] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 621.873676] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 621.873880] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 621.874082] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 621.874277] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 621.874441] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 621.874913] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 621.885979] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 621.886226] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 621.886401] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 621.886557] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 621.887659] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00c25f9b-1ed3-4548-8154-0aecf7b8942d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.897648] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d8423ae-2c2c-4cc7-b159-799d6970f9f3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.914349] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e54396b4-2821-45fd-88c6-a9a25a8ae2fd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.921873] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5c2093ed-cdfd-4278-a588-bc5f1141aae8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.953241] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180525MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 621.953421] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 621.953628] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.063796] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 6699f6e7-203d-4d8a-bc9e-8c8d5427db36 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.091038] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4846a37e-b175-4f23-b8c1-34a6b124513d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.102780] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 811bed85-6a4d-4340-9e4f-a2acd86d921c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.114831] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance cc6f5d6c-1f3c-4ead-8ee4-9b59923ac57b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.125681] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77493b32-817c-4be8-a42d-8d48e707d41f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.136214] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 362b7115-a270-407f-a53c-506c863c8fe1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.146300] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 68fe9a49-e561-45da-afcb-e804e2109e76 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.158737] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8728833b-c3ac-49b2-9a33-ecfae9597e9e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.172428] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8dd5ad91-d09a-41dd-a8d0-fd22e96fc279 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.183567] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c936b733-7faa-4e73-99c5-2ac11d5421ff has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.183769] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f1dddee3-33d6-4f55-bd91-9ce2fe6be72b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 622.213187] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance ff6cf180-c0fe-4b4f-9bac-08e9e3dc7c1d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.213187] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 622.213187] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance ab9cf87a-b74f-4b2f-842d-26e16b7fdb92 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 622.213187] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 94d19d7f-8eb7-4c93-aff4-3b1404338240 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 622.225778] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7d000257-5fc1-43fa-9fc5-a07519a6feea has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.226638] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 622.238109] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.238292] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 02ed8dca-f6ae-47b0-a047-f89f858f5ff2 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 622.258992] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f889087d-5f22-4de6-8dff-819d2afb7240 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.271871] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 3af8fe9a-643b-4650-a036-6f1b9c3c8d03 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.286884] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.297847] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1979e66a-172d-4130-90ea-9f0034b13071 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.309406] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 6b48c313-e3d5-4d3c-83c5-3c12d948abba has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.322413] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 11838596-1730-4c23-bbf2-3159d3c6d72d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.333805] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 707e02eb-1471-4ddd-be9e-a41f33d42482 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.334067] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 11a32984-5548-4db4-8788-9b221ba381df actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 622.344634] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a4e56801-a8b8-448f-9b1a-d228ceeaaa46 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.355400] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 717fd312-ac1d-42b6-9f93-13b9f235bd23 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.355552] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5501c192-867b-40c0-9bea-c3c44865d2f4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 622.355674] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0952267f-c258-4575-be86-b2bc48e9863b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 622.370333] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a12ff1bf-8cfa-4d03-828d-6b34727f61bf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.370539] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a5c37718-0067-4f5e-ba7e-533f13529739 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 622.384048] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 622.384336] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 622.384474] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '1', 'num_instances': '10', 'num_vm_building': '10', 'num_task_spawning': '10', 'num_os_type_None': '10', 'num_proj_6a04825b924a482994b31b7066af5714': '1', 'io_workload': '10', 'num_proj_726f39967ec7455dadc4ed498eff094b': '1', 'num_proj_c0deaad067fd40d684e8cecddd605788': '1', 'num_proj_37d368bc456740739821f51e3a25f2fb': '1', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'num_proj_93e7b65b07c44f5c88154e6f398bdbd8': '1', 'num_proj_d806b4c5831b457fa075e387f55d8ad5': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_4b844f0a784d4f20a5adbf7354c75cc7': '1', 'num_proj_ad03061e544f4b17abb83d9596c461a0': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 622.837628] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b56dcc4b-868b-49fe-a963-23af923c5e33 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.846454] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e471631a-ae7b-4d5e-8169-c4dfde7cd9bc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.877513] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37d502e1-4e27-4f89-af67-8cef67dea25d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.887506] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59b4d089-7a38-4b0d-981f-47d532260829 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.899944] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 622.910031] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 622.944035] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 622.944216] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.991s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.962467] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 622.962725] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 622.962899] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 627.228566] env[63273]: DEBUG oslo_concurrency.lockutils [None req-92deb72e-507e-4804-9869-474be9cc2156 tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Acquiring lock "02e9cc93-bdc3-4220-a22e-e29e959a5377" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.229114] env[63273]: DEBUG oslo_concurrency.lockutils [None req-92deb72e-507e-4804-9869-474be9cc2156 tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Lock "02e9cc93-bdc3-4220-a22e-e29e959a5377" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.263719] env[63273]: DEBUG oslo_concurrency.lockutils [None req-92deb72e-507e-4804-9869-474be9cc2156 tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Acquiring lock "451c95b6-a407-4911-bc40-83f2e2e7a75f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.263719] env[63273]: DEBUG oslo_concurrency.lockutils [None req-92deb72e-507e-4804-9869-474be9cc2156 tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Lock "451c95b6-a407-4911-bc40-83f2e2e7a75f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 648.248852] env[63273]: WARNING oslo_vmware.rw_handles [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 648.248852] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 648.248852] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 648.248852] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 648.248852] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 648.248852] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 648.248852] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 648.248852] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 648.248852] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 648.248852] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 648.248852] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 648.248852] env[63273]: ERROR oslo_vmware.rw_handles [ 648.249551] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/b32b776b-fcca-4355-8b63-5164e7798599/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 648.251028] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 648.251293] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Copying Virtual Disk [datastore1] vmware_temp/b32b776b-fcca-4355-8b63-5164e7798599/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/b32b776b-fcca-4355-8b63-5164e7798599/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 648.251606] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-ffa871c0-a2ef-45f4-81ba-4a0c098b0686 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.259654] env[63273]: DEBUG oslo_vmware.api [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Waiting for the task: (returnval){ [ 648.259654] env[63273]: value = "task-5072020" [ 648.259654] env[63273]: _type = "Task" [ 648.259654] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 648.268304] env[63273]: DEBUG oslo_vmware.api [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Task: {'id': task-5072020, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 648.770769] env[63273]: DEBUG oslo_vmware.exceptions [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 648.771119] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 648.771850] env[63273]: ERROR nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 648.771850] env[63273]: Faults: ['InvalidArgument'] [ 648.771850] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Traceback (most recent call last): [ 648.771850] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 648.771850] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] yield resources [ 648.771850] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 648.771850] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] self.driver.spawn(context, instance, image_meta, [ 648.771850] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 648.771850] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 648.771850] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 648.771850] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] self._fetch_image_if_missing(context, vi) [ 648.771850] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] image_cache(vi, tmp_image_ds_loc) [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] vm_util.copy_virtual_disk( [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] session._wait_for_task(vmdk_copy_task) [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] return self.wait_for_task(task_ref) [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] return evt.wait() [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] result = hub.switch() [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 648.772342] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] return self.greenlet.switch() [ 648.772831] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 648.772831] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] self.f(*self.args, **self.kw) [ 648.772831] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 648.772831] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] raise exceptions.translate_fault(task_info.error) [ 648.772831] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 648.772831] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Faults: ['InvalidArgument'] [ 648.772831] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] [ 648.772831] env[63273]: INFO nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Terminating instance [ 648.774016] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 648.774105] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 648.774296] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a454c5c6-cc94-49b9-8920-dbef28a4ce0b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.777176] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquiring lock "refresh_cache-0952267f-c258-4575-be86-b2bc48e9863b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 648.777331] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquired lock "refresh_cache-0952267f-c258-4575-be86-b2bc48e9863b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 648.777504] env[63273]: DEBUG nova.network.neutron [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 648.782794] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 648.783948] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 648.783948] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8f256d55-8611-47c7-87f4-1b5a8acbbf0a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.791166] env[63273]: DEBUG oslo_vmware.api [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Waiting for the task: (returnval){ [ 648.791166] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52d43a7e-482c-2b8d-b4bd-5445da0f1d13" [ 648.791166] env[63273]: _type = "Task" [ 648.791166] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 648.803096] env[63273]: DEBUG oslo_vmware.api [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52d43a7e-482c-2b8d-b4bd-5445da0f1d13, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 648.811674] env[63273]: DEBUG nova.network.neutron [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 648.908788] env[63273]: DEBUG nova.network.neutron [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 648.918971] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Releasing lock "refresh_cache-0952267f-c258-4575-be86-b2bc48e9863b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 648.919406] env[63273]: DEBUG nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 648.919617] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 648.920782] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-435afd89-2639-4da6-8ea7-079e03a23ea4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.929371] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 648.929602] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-b401ed12-d2dc-4b46-a857-3bff96d95c4b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.972232] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 648.972631] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 648.972867] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Deleting the datastore file [datastore1] 0952267f-c258-4575-be86-b2bc48e9863b {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 648.973226] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-6e8b3a1c-bfbd-4d2b-b819-cf226385fffa {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.980574] env[63273]: DEBUG oslo_vmware.api [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Waiting for the task: (returnval){ [ 648.980574] env[63273]: value = "task-5072022" [ 648.980574] env[63273]: _type = "Task" [ 648.980574] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 648.989469] env[63273]: DEBUG oslo_vmware.api [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Task: {'id': task-5072022, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 649.303087] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 649.303471] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Creating directory with path [datastore1] vmware_temp/28efd023-0097-4b44-985c-67c226c299a6/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 649.303471] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-c42e2426-39e2-4324-af85-9b44718ddd48 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.324156] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Created directory with path [datastore1] vmware_temp/28efd023-0097-4b44-985c-67c226c299a6/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 649.324441] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Fetch image to [datastore1] vmware_temp/28efd023-0097-4b44-985c-67c226c299a6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 649.324619] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/28efd023-0097-4b44-985c-67c226c299a6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 649.325537] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5f2c0e8-1624-4993-9656-89ca4f0a0701 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.333315] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a4062b1-22da-4171-9b01-a39766f3968d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.342704] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8db653db-96e9-4ab3-b2fd-5a18e88ec078 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.378010] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a829d42-107d-4849-8a4e-fd65ec7eedec {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.386216] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-12f2b844-8da3-4be3-b2f5-a5ead1c71e0c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.409463] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 649.472263] env[63273]: DEBUG oslo_vmware.rw_handles [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/28efd023-0097-4b44-985c-67c226c299a6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 649.530996] env[63273]: DEBUG oslo_vmware.rw_handles [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 649.531219] env[63273]: DEBUG oslo_vmware.rw_handles [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/28efd023-0097-4b44-985c-67c226c299a6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 649.535675] env[63273]: DEBUG oslo_vmware.api [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Task: {'id': task-5072022, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.034216} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 649.535921] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 649.536113] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 649.536360] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 649.536563] env[63273]: INFO nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Took 0.62 seconds to destroy the instance on the hypervisor. [ 649.536819] env[63273]: DEBUG oslo.service.loopingcall [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 649.537040] env[63273]: DEBUG nova.compute.manager [-] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Skipping network deallocation for instance since networking was not requested. {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2277}} [ 649.539662] env[63273]: DEBUG nova.compute.claims [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 649.539895] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 649.540173] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.056012] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-800b00f8-a6dc-471b-8df5-ffd3d77f5e7a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.063745] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be7c07cc-726a-4425-8883-24809499dee1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.094514] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ff506b2-b1e9-4740-b373-6f252240e006 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.102527] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1510d95b-0c60-457a-a7fe-06cc864d1db2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.117138] env[63273]: DEBUG nova.compute.provider_tree [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 650.126691] env[63273]: DEBUG nova.scheduler.client.report [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 650.142105] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.602s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.142738] env[63273]: ERROR nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 650.142738] env[63273]: Faults: ['InvalidArgument'] [ 650.142738] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Traceback (most recent call last): [ 650.142738] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 650.142738] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] self.driver.spawn(context, instance, image_meta, [ 650.142738] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 650.142738] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 650.142738] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 650.142738] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] self._fetch_image_if_missing(context, vi) [ 650.142738] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 650.142738] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] image_cache(vi, tmp_image_ds_loc) [ 650.142738] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] vm_util.copy_virtual_disk( [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] session._wait_for_task(vmdk_copy_task) [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] return self.wait_for_task(task_ref) [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] return evt.wait() [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] result = hub.switch() [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] return self.greenlet.switch() [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 650.143187] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] self.f(*self.args, **self.kw) [ 650.143624] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 650.143624] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] raise exceptions.translate_fault(task_info.error) [ 650.143624] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 650.143624] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Faults: ['InvalidArgument'] [ 650.143624] env[63273]: ERROR nova.compute.manager [instance: 0952267f-c258-4575-be86-b2bc48e9863b] [ 650.143624] env[63273]: DEBUG nova.compute.utils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 650.146618] env[63273]: DEBUG nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Build of instance 0952267f-c258-4575-be86-b2bc48e9863b was re-scheduled: A specified parameter was not correct: fileType [ 650.146618] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 650.147032] env[63273]: DEBUG nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 650.147262] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquiring lock "refresh_cache-0952267f-c258-4575-be86-b2bc48e9863b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 650.147408] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Acquired lock "refresh_cache-0952267f-c258-4575-be86-b2bc48e9863b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 650.147567] env[63273]: DEBUG nova.network.neutron [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 650.173825] env[63273]: DEBUG nova.network.neutron [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 650.281948] env[63273]: DEBUG nova.network.neutron [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 650.294638] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Releasing lock "refresh_cache-0952267f-c258-4575-be86-b2bc48e9863b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 650.295336] env[63273]: DEBUG nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 650.295336] env[63273]: DEBUG nova.compute.manager [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] Skipping network deallocation for instance since networking was not requested. {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2277}} [ 650.405518] env[63273]: INFO nova.scheduler.client.report [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Deleted allocations for instance 0952267f-c258-4575-be86-b2bc48e9863b [ 650.431124] env[63273]: DEBUG oslo_concurrency.lockutils [None req-246022ca-eec9-4aef-8a6b-f93cf65d0c10 tempest-ServerDiagnosticsV248Test-1939174066 tempest-ServerDiagnosticsV248Test-1939174066-project-member] Lock "0952267f-c258-4575-be86-b2bc48e9863b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 101.501s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.432441] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "0952267f-c258-4575-be86-b2bc48e9863b" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 86.157s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.432724] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0952267f-c258-4575-be86-b2bc48e9863b] During sync_power_state the instance has a pending task (spawning). Skip. [ 650.432895] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "0952267f-c258-4575-be86-b2bc48e9863b" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.457032] env[63273]: DEBUG nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 650.513289] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.513588] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.515455] env[63273]: INFO nova.compute.claims [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 650.999797] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dcc54c1e-0ac8-46a5-aa50-8490fe5ce89c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.008450] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0534bf95-80bb-4477-aa30-311454a2c739 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.040639] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8068d4f8-764c-49db-b8d4-d38913c7e53b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.048664] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbb2e8ee-c988-4197-a0fa-7103de61c02e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.062220] env[63273]: DEBUG nova.compute.provider_tree [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 651.071050] env[63273]: DEBUG nova.scheduler.client.report [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 651.090349] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.577s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 651.090844] env[63273]: DEBUG nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 651.124806] env[63273]: DEBUG nova.compute.utils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 651.126468] env[63273]: DEBUG nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 651.126684] env[63273]: DEBUG nova.network.neutron [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 651.139671] env[63273]: DEBUG nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 651.204725] env[63273]: DEBUG nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 651.207773] env[63273]: DEBUG nova.policy [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '269b0c06a714425fa133f94b9293bf0a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '06e0618687eb4144b9aacdc90a42879d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 651.240035] env[63273]: DEBUG nova.virt.hardware [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 651.240830] env[63273]: DEBUG nova.virt.hardware [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 651.240830] env[63273]: DEBUG nova.virt.hardware [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 651.240830] env[63273]: DEBUG nova.virt.hardware [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 651.240830] env[63273]: DEBUG nova.virt.hardware [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 651.241029] env[63273]: DEBUG nova.virt.hardware [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 651.241142] env[63273]: DEBUG nova.virt.hardware [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 651.241314] env[63273]: DEBUG nova.virt.hardware [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 651.241618] env[63273]: DEBUG nova.virt.hardware [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 651.241823] env[63273]: DEBUG nova.virt.hardware [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 651.242080] env[63273]: DEBUG nova.virt.hardware [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 651.243282] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4167c4c3-0ce4-4791-a3f7-4c7bb5905777 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.251747] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d97c761-cfa1-4827-9960-62094f638f50 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.674362] env[63273]: DEBUG nova.network.neutron [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Successfully created port: 5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 652.760158] env[63273]: DEBUG nova.network.neutron [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Successfully updated port: 5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 652.778052] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquiring lock "refresh_cache-77493b32-817c-4be8-a42d-8d48e707d41f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 652.778214] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquired lock "refresh_cache-77493b32-817c-4be8-a42d-8d48e707d41f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 652.778369] env[63273]: DEBUG nova.network.neutron [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 652.792845] env[63273]: DEBUG nova.compute.manager [req-98b68870-f528-42e5-8431-417835748f94 req-22411a76-f424-4449-a0e3-cc83a8b538c0 service nova] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Received event network-vif-plugged-5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 652.793035] env[63273]: DEBUG oslo_concurrency.lockutils [req-98b68870-f528-42e5-8431-417835748f94 req-22411a76-f424-4449-a0e3-cc83a8b538c0 service nova] Acquiring lock "77493b32-817c-4be8-a42d-8d48e707d41f-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 652.793263] env[63273]: DEBUG oslo_concurrency.lockutils [req-98b68870-f528-42e5-8431-417835748f94 req-22411a76-f424-4449-a0e3-cc83a8b538c0 service nova] Lock "77493b32-817c-4be8-a42d-8d48e707d41f-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 652.793435] env[63273]: DEBUG oslo_concurrency.lockutils [req-98b68870-f528-42e5-8431-417835748f94 req-22411a76-f424-4449-a0e3-cc83a8b538c0 service nova] Lock "77493b32-817c-4be8-a42d-8d48e707d41f-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 652.793609] env[63273]: DEBUG nova.compute.manager [req-98b68870-f528-42e5-8431-417835748f94 req-22411a76-f424-4449-a0e3-cc83a8b538c0 service nova] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] No waiting events found dispatching network-vif-plugged-5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 652.793798] env[63273]: WARNING nova.compute.manager [req-98b68870-f528-42e5-8431-417835748f94 req-22411a76-f424-4449-a0e3-cc83a8b538c0 service nova] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Received unexpected event network-vif-plugged-5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca for instance with vm_state building and task_state spawning. [ 652.844874] env[63273]: DEBUG nova.network.neutron [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 653.098866] env[63273]: DEBUG nova.network.neutron [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Updating instance_info_cache with network_info: [{"id": "5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca", "address": "fa:16:3e:82:62:5f", "network": {"id": "194c7166-1bca-4ca4-9c1f-5fb45bd5feb3", "bridge": "br-int", "label": "tempest-InstanceActionsTestJSON-923353737-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "06e0618687eb4144b9aacdc90a42879d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "01fe2e08-46f6-4cee-aefd-934461f8077d", "external-id": "nsx-vlan-transportzone-806", "segmentation_id": 806, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5cdfc8b1-6a", "ovs_interfaceid": "5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 653.111380] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Releasing lock "refresh_cache-77493b32-817c-4be8-a42d-8d48e707d41f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 653.111724] env[63273]: DEBUG nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Instance network_info: |[{"id": "5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca", "address": "fa:16:3e:82:62:5f", "network": {"id": "194c7166-1bca-4ca4-9c1f-5fb45bd5feb3", "bridge": "br-int", "label": "tempest-InstanceActionsTestJSON-923353737-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "06e0618687eb4144b9aacdc90a42879d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "01fe2e08-46f6-4cee-aefd-934461f8077d", "external-id": "nsx-vlan-transportzone-806", "segmentation_id": 806, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5cdfc8b1-6a", "ovs_interfaceid": "5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 653.112187] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:82:62:5f', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '01fe2e08-46f6-4cee-aefd-934461f8077d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 653.119927] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Creating folder: Project (06e0618687eb4144b9aacdc90a42879d). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 653.121364] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-6cd60380-3ada-4c07-a6f3-ddbdd6dc1fad {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.136624] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Created folder: Project (06e0618687eb4144b9aacdc90a42879d) in parent group-v986930. [ 653.136624] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Creating folder: Instances. Parent ref: group-v986968. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 653.136624] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-4e5c8e6b-6793-4b61-ac33-a49d6aa54ba9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.146668] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Created folder: Instances in parent group-v986968. [ 653.146797] env[63273]: DEBUG oslo.service.loopingcall [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 653.146994] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 653.147221] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-1db80d9e-a50b-4ef1-b8e7-042b1824d73f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.167441] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 653.167441] env[63273]: value = "task-5072025" [ 653.167441] env[63273]: _type = "Task" [ 653.167441] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 653.175461] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072025, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 653.682029] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072025, 'name': CreateVM_Task, 'duration_secs': 0.342065} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 653.682029] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 653.682029] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 653.682029] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 653.682029] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 653.682408] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-ace95da4-a0f8-4c51-9db0-af15c8f5f71f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.687946] env[63273]: DEBUG oslo_vmware.api [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Waiting for the task: (returnval){ [ 653.687946] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]527efdfc-7dab-2291-3d23-28c16305606b" [ 653.687946] env[63273]: _type = "Task" [ 653.687946] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 653.696878] env[63273]: DEBUG oslo_vmware.api [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]527efdfc-7dab-2291-3d23-28c16305606b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 654.201091] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 654.201926] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 654.202271] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 655.004998] env[63273]: DEBUG nova.compute.manager [req-d8e27d98-d3a3-4aa4-a654-e5c615ffdde4 req-e2db6a42-b9e1-4543-bf46-8e89efa4f309 service nova] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Received event network-changed-5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 655.005204] env[63273]: DEBUG nova.compute.manager [req-d8e27d98-d3a3-4aa4-a654-e5c615ffdde4 req-e2db6a42-b9e1-4543-bf46-8e89efa4f309 service nova] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Refreshing instance network info cache due to event network-changed-5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 655.005861] env[63273]: DEBUG oslo_concurrency.lockutils [req-d8e27d98-d3a3-4aa4-a654-e5c615ffdde4 req-e2db6a42-b9e1-4543-bf46-8e89efa4f309 service nova] Acquiring lock "refresh_cache-77493b32-817c-4be8-a42d-8d48e707d41f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 655.006032] env[63273]: DEBUG oslo_concurrency.lockutils [req-d8e27d98-d3a3-4aa4-a654-e5c615ffdde4 req-e2db6a42-b9e1-4543-bf46-8e89efa4f309 service nova] Acquired lock "refresh_cache-77493b32-817c-4be8-a42d-8d48e707d41f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 655.006246] env[63273]: DEBUG nova.network.neutron [req-d8e27d98-d3a3-4aa4-a654-e5c615ffdde4 req-e2db6a42-b9e1-4543-bf46-8e89efa4f309 service nova] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Refreshing network info cache for port 5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 655.829940] env[63273]: DEBUG nova.network.neutron [req-d8e27d98-d3a3-4aa4-a654-e5c615ffdde4 req-e2db6a42-b9e1-4543-bf46-8e89efa4f309 service nova] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Updated VIF entry in instance network info cache for port 5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 655.830319] env[63273]: DEBUG nova.network.neutron [req-d8e27d98-d3a3-4aa4-a654-e5c615ffdde4 req-e2db6a42-b9e1-4543-bf46-8e89efa4f309 service nova] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Updating instance_info_cache with network_info: [{"id": "5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca", "address": "fa:16:3e:82:62:5f", "network": {"id": "194c7166-1bca-4ca4-9c1f-5fb45bd5feb3", "bridge": "br-int", "label": "tempest-InstanceActionsTestJSON-923353737-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "06e0618687eb4144b9aacdc90a42879d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "01fe2e08-46f6-4cee-aefd-934461f8077d", "external-id": "nsx-vlan-transportzone-806", "segmentation_id": 806, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5cdfc8b1-6a", "ovs_interfaceid": "5cdfc8b1-6a35-4b48-a48a-73cc7bcfb5ca", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.839767] env[63273]: DEBUG oslo_concurrency.lockutils [req-d8e27d98-d3a3-4aa4-a654-e5c615ffdde4 req-e2db6a42-b9e1-4543-bf46-8e89efa4f309 service nova] Releasing lock "refresh_cache-77493b32-817c-4be8-a42d-8d48e707d41f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 678.695510] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquiring lock "e36789b5-f814-4105-b144-361fef9e0d0e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 678.695776] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Lock "e36789b5-f814-4105-b144-361fef9e0d0e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 680.892083] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 680.892468] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 680.892747] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 680.892985] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 680.893251] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 680.907880] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 680.908370] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 680.908370] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 680.908431] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 680.909557] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-edaea2e7-abce-428e-8c64-005734ad6fb9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 680.921030] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52437f49-ba73-4ba3-987a-d42da50647e9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 680.936494] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64459d45-3f17-48b0-a387-a690e837f8e1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 680.944587] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eca73ee2-aff8-4d2c-8bcb-d117413fbd87 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 680.975246] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180546MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 680.975406] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 680.975606] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 681.069846] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 6699f6e7-203d-4d8a-bc9e-8c8d5427db36 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.081063] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4846a37e-b175-4f23-b8c1-34a6b124513d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.092468] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 811bed85-6a4d-4340-9e4f-a2acd86d921c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.109656] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance cc6f5d6c-1f3c-4ead-8ee4-9b59923ac57b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.109833] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77493b32-817c-4be8-a42d-8d48e707d41f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 681.121545] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 362b7115-a270-407f-a53c-506c863c8fe1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.132180] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 68fe9a49-e561-45da-afcb-e804e2109e76 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.142319] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8728833b-c3ac-49b2-9a33-ecfae9597e9e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.153241] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 02e9cc93-bdc3-4220-a22e-e29e959a5377 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.165993] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 451c95b6-a407-4911-bc40-83f2e2e7a75f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.177352] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8dd5ad91-d09a-41dd-a8d0-fd22e96fc279 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.190189] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c936b733-7faa-4e73-99c5-2ac11d5421ff has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.190189] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f1dddee3-33d6-4f55-bd91-9ce2fe6be72b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 681.201713] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance ff6cf180-c0fe-4b4f-9bac-08e9e3dc7c1d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.201903] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 681.202104] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance ab9cf87a-b74f-4b2f-842d-26e16b7fdb92 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 681.202234] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 94d19d7f-8eb7-4c93-aff4-3b1404338240 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 681.212876] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7d000257-5fc1-43fa-9fc5-a07519a6feea has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.213029] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 681.223395] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.223574] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 02ed8dca-f6ae-47b0-a047-f89f858f5ff2 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 681.234173] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f889087d-5f22-4de6-8dff-819d2afb7240 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.245852] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 3af8fe9a-643b-4650-a036-6f1b9c3c8d03 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.258034] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.268957] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1979e66a-172d-4130-90ea-9f0034b13071 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.280171] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 6b48c313-e3d5-4d3c-83c5-3c12d948abba has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.291689] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 11838596-1730-4c23-bbf2-3159d3c6d72d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.302743] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 707e02eb-1471-4ddd-be9e-a41f33d42482 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.302895] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 11a32984-5548-4db4-8788-9b221ba381df actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 681.313898] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a4e56801-a8b8-448f-9b1a-d228ceeaaa46 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.325034] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 717fd312-ac1d-42b6-9f93-13b9f235bd23 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.325034] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5501c192-867b-40c0-9bea-c3c44865d2f4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 681.336284] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e36789b5-f814-4105-b144-361fef9e0d0e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.348996] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a12ff1bf-8cfa-4d03-828d-6b34727f61bf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.349172] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a5c37718-0067-4f5e-ba7e-533f13529739 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 681.359692] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 681.359958] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 681.360163] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '2', 'num_instances': '10', 'num_vm_building': '10', 'num_task_spawning': '10', 'num_os_type_None': '10', 'num_proj_6a04825b924a482994b31b7066af5714': '1', 'io_workload': '10', 'num_proj_726f39967ec7455dadc4ed498eff094b': '1', 'num_proj_37d368bc456740739821f51e3a25f2fb': '1', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'num_proj_93e7b65b07c44f5c88154e6f398bdbd8': '1', 'num_proj_d806b4c5831b457fa075e387f55d8ad5': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_4b844f0a784d4f20a5adbf7354c75cc7': '1', 'num_proj_ad03061e544f4b17abb83d9596c461a0': '1', 'num_proj_06e0618687eb4144b9aacdc90a42879d': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 681.803466] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b920658-f661-45bc-96b9-316832f51f2e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.812267] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0655f3f1-ecbb-4ddb-8207-c8e08961bcfe {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.843029] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e033ace2-32c7-45dd-ba73-2fa34f09f410 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.850519] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05a83090-7e24-48be-9e91-7b2c0ae65795 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.863619] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 681.872107] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 681.887043] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 681.887224] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.912s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 682.886417] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 682.886747] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 682.891214] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 682.891381] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 682.891502] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 682.912361] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 682.912568] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 682.912738] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 682.912874] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 682.912998] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 682.913147] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 682.913274] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 682.913398] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 682.913515] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 682.913633] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 682.913751] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 682.914264] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 682.914437] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 698.268107] env[63273]: WARNING oslo_vmware.rw_handles [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 698.268107] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 698.268107] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 698.268107] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 698.268107] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 698.268107] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 698.268107] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 698.268107] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 698.268107] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 698.268107] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 698.268107] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 698.268107] env[63273]: ERROR oslo_vmware.rw_handles [ 698.268762] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/28efd023-0097-4b44-985c-67c226c299a6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 698.270209] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 698.270457] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Copying Virtual Disk [datastore1] vmware_temp/28efd023-0097-4b44-985c-67c226c299a6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/28efd023-0097-4b44-985c-67c226c299a6/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 698.270740] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-e3af785b-7ec0-4c60-9329-5f5e01cac70f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.279170] env[63273]: DEBUG oslo_vmware.api [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Waiting for the task: (returnval){ [ 698.279170] env[63273]: value = "task-5072026" [ 698.279170] env[63273]: _type = "Task" [ 698.279170] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 698.288469] env[63273]: DEBUG oslo_vmware.api [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Task: {'id': task-5072026, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 698.790167] env[63273]: DEBUG oslo_vmware.exceptions [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 698.790458] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 698.791016] env[63273]: ERROR nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 698.791016] env[63273]: Faults: ['InvalidArgument'] [ 698.791016] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Traceback (most recent call last): [ 698.791016] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 698.791016] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] yield resources [ 698.791016] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 698.791016] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] self.driver.spawn(context, instance, image_meta, [ 698.791016] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 698.791016] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 698.791016] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 698.791016] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] self._fetch_image_if_missing(context, vi) [ 698.791016] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] image_cache(vi, tmp_image_ds_loc) [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] vm_util.copy_virtual_disk( [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] session._wait_for_task(vmdk_copy_task) [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] return self.wait_for_task(task_ref) [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] return evt.wait() [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] result = hub.switch() [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 698.791426] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] return self.greenlet.switch() [ 698.791821] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 698.791821] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] self.f(*self.args, **self.kw) [ 698.791821] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 698.791821] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] raise exceptions.translate_fault(task_info.error) [ 698.791821] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 698.791821] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Faults: ['InvalidArgument'] [ 698.791821] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] [ 698.791821] env[63273]: INFO nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Terminating instance [ 698.793708] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 698.793708] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 698.794473] env[63273]: DEBUG nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 698.794473] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 698.794793] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-3e4e5e0c-c650-4119-8fd4-773c532548c4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.797175] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b882a85-19cd-445b-8124-26a39e144ede {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.805197] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 698.805431] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-a88a6db1-d886-410f-9a66-932e641579d6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.807810] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 698.808467] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 698.808961] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-84efa508-6ad9-4813-8529-d1b485896c38 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.814159] env[63273]: DEBUG oslo_vmware.api [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Waiting for the task: (returnval){ [ 698.814159] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ca2fa0-c634-ef17-9921-d6271633b165" [ 698.814159] env[63273]: _type = "Task" [ 698.814159] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 698.822263] env[63273]: DEBUG oslo_vmware.api [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ca2fa0-c634-ef17-9921-d6271633b165, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 698.876185] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 698.876419] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 698.876589] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Deleting the datastore file [datastore1] 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 698.876888] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-ea304aaf-873e-4f88-a292-7079d32c49c0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.884835] env[63273]: DEBUG oslo_vmware.api [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Waiting for the task: (returnval){ [ 698.884835] env[63273]: value = "task-5072028" [ 698.884835] env[63273]: _type = "Task" [ 698.884835] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 698.894704] env[63273]: DEBUG oslo_vmware.api [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Task: {'id': task-5072028, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 699.325350] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 699.325659] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Creating directory with path [datastore1] vmware_temp/054a93d6-af3b-4ee1-beb7-23bd6478ba61/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 699.325835] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-cb5c0b41-52cb-4483-8dc5-ca0ec2ce8091 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.338040] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Created directory with path [datastore1] vmware_temp/054a93d6-af3b-4ee1-beb7-23bd6478ba61/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 699.338177] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Fetch image to [datastore1] vmware_temp/054a93d6-af3b-4ee1-beb7-23bd6478ba61/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 699.338351] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/054a93d6-af3b-4ee1-beb7-23bd6478ba61/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 699.339118] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08faed92-132b-4aee-91a4-9f9bb588e29d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.346285] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39c4c62a-62cd-4722-8822-f994d5bcf694 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.357041] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9ed13b0-5dcc-4958-bc02-5e6c5ba7f28a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.390941] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5138a932-3f09-4dcf-aec0-398477bfd181 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.400438] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-a41fde17-3b10-4962-8e0d-7c06b697f5fe {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.402245] env[63273]: DEBUG oslo_vmware.api [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Task: {'id': task-5072028, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.081642} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 699.402499] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 699.402678] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 699.402848] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 699.403028] env[63273]: INFO nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Took 0.61 seconds to destroy the instance on the hypervisor. [ 699.405546] env[63273]: DEBUG nova.compute.claims [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 699.405722] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 699.405935] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 699.426785] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 699.497843] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/054a93d6-af3b-4ee1-beb7-23bd6478ba61/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 699.557206] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 699.558115] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/054a93d6-af3b-4ee1-beb7-23bd6478ba61/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 699.974150] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8941d480-3913-4970-a099-b8ddad9b1203 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.983042] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86201554-c0bf-4247-a5ed-6082bf0a77e4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 700.015890] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70757ad5-ca98-42b8-a25d-a51a4d8d9ef3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 700.025026] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f58d8aca-e727-4b75-869e-24779ad27d3a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 700.039319] env[63273]: DEBUG nova.compute.provider_tree [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 700.051608] env[63273]: DEBUG nova.scheduler.client.report [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 700.070034] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.664s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 700.070636] env[63273]: ERROR nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 700.070636] env[63273]: Faults: ['InvalidArgument'] [ 700.070636] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Traceback (most recent call last): [ 700.070636] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 700.070636] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] self.driver.spawn(context, instance, image_meta, [ 700.070636] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 700.070636] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 700.070636] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 700.070636] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] self._fetch_image_if_missing(context, vi) [ 700.070636] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 700.070636] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] image_cache(vi, tmp_image_ds_loc) [ 700.070636] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] vm_util.copy_virtual_disk( [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] session._wait_for_task(vmdk_copy_task) [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] return self.wait_for_task(task_ref) [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] return evt.wait() [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] result = hub.switch() [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] return self.greenlet.switch() [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 700.070974] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] self.f(*self.args, **self.kw) [ 700.071431] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 700.071431] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] raise exceptions.translate_fault(task_info.error) [ 700.071431] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 700.071431] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Faults: ['InvalidArgument'] [ 700.071431] env[63273]: ERROR nova.compute.manager [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] [ 700.071431] env[63273]: DEBUG nova.compute.utils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 700.073111] env[63273]: DEBUG nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Build of instance 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5 was re-scheduled: A specified parameter was not correct: fileType [ 700.073111] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 700.073493] env[63273]: DEBUG nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 700.073665] env[63273]: DEBUG nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 700.073819] env[63273]: DEBUG nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 700.073982] env[63273]: DEBUG nova.network.neutron [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 700.525118] env[63273]: DEBUG nova.network.neutron [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 700.613753] env[63273]: INFO nova.compute.manager [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] Took 0.54 seconds to deallocate network for instance. [ 700.720575] env[63273]: INFO nova.scheduler.client.report [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Deleted allocations for instance 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5 [ 700.748030] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d94fc961-1903-42e0-a8c5-ac684c2d8bf5 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 153.757s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 700.749441] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 136.475s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 700.749985] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4afbddcb-d9ac-412e-93d9-52b2de6fbdb5] During sync_power_state the instance has a pending task (spawning). Skip. [ 700.750197] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "4afbddcb-d9ac-412e-93d9-52b2de6fbdb5" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 700.775403] env[63273]: DEBUG nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 700.830546] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 700.830842] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 700.832608] env[63273]: INFO nova.compute.claims [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 701.328196] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f09619c-0434-45f3-8b39-ea2308a00296 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.336697] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4c7135f5-9051-4855-a213-1bc2a5cac5d7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.367655] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc6f3e3d-6807-4132-acf2-305d65d9abc0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.375200] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bfdcfc02-db8d-41fd-82a7-293e3458abd6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.389096] env[63273]: DEBUG nova.compute.provider_tree [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 701.401361] env[63273]: DEBUG nova.scheduler.client.report [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 701.417492] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.587s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 701.417998] env[63273]: DEBUG nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 701.457868] env[63273]: DEBUG nova.compute.utils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 701.459204] env[63273]: DEBUG nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 701.460029] env[63273]: DEBUG nova.network.neutron [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 701.469122] env[63273]: DEBUG nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 701.537025] env[63273]: DEBUG nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 701.550160] env[63273]: DEBUG nova.policy [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '62bb56013a3144bcbc1e75be179b6380', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '70158b9660d14db684be2828e94f8e42', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 701.565064] env[63273]: DEBUG nova.virt.hardware [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 701.565325] env[63273]: DEBUG nova.virt.hardware [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 701.565482] env[63273]: DEBUG nova.virt.hardware [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 701.565661] env[63273]: DEBUG nova.virt.hardware [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 701.565806] env[63273]: DEBUG nova.virt.hardware [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 701.565950] env[63273]: DEBUG nova.virt.hardware [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 701.566967] env[63273]: DEBUG nova.virt.hardware [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 701.567326] env[63273]: DEBUG nova.virt.hardware [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 701.567516] env[63273]: DEBUG nova.virt.hardware [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 701.567692] env[63273]: DEBUG nova.virt.hardware [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 701.567869] env[63273]: DEBUG nova.virt.hardware [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 701.568865] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1315f45-a7d9-4f1b-83ba-8a538bbbb062 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.578806] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eda87473-bb1b-4340-a53d-b856a203307a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.885823] env[63273]: DEBUG nova.network.neutron [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Successfully created port: 9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 702.869931] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 702.869931] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 702.961781] env[63273]: DEBUG nova.network.neutron [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Successfully updated port: 9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 702.972323] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquiring lock "refresh_cache-f4d86853-25c8-4568-9b3f-8f07bcc5f068" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 702.973085] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquired lock "refresh_cache-f4d86853-25c8-4568-9b3f-8f07bcc5f068" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 702.973085] env[63273]: DEBUG nova.network.neutron [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 703.059748] env[63273]: DEBUG nova.network.neutron [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 703.226206] env[63273]: DEBUG nova.compute.manager [req-c10238b8-ebe7-4677-90a3-2c5d4e4c78c0 req-3622d933-a072-4347-9fa1-8fe4919610b8 service nova] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Received event network-vif-plugged-9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 703.226206] env[63273]: DEBUG oslo_concurrency.lockutils [req-c10238b8-ebe7-4677-90a3-2c5d4e4c78c0 req-3622d933-a072-4347-9fa1-8fe4919610b8 service nova] Acquiring lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.226828] env[63273]: DEBUG oslo_concurrency.lockutils [req-c10238b8-ebe7-4677-90a3-2c5d4e4c78c0 req-3622d933-a072-4347-9fa1-8fe4919610b8 service nova] Lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.227298] env[63273]: DEBUG oslo_concurrency.lockutils [req-c10238b8-ebe7-4677-90a3-2c5d4e4c78c0 req-3622d933-a072-4347-9fa1-8fe4919610b8 service nova] Lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.002s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.227584] env[63273]: DEBUG nova.compute.manager [req-c10238b8-ebe7-4677-90a3-2c5d4e4c78c0 req-3622d933-a072-4347-9fa1-8fe4919610b8 service nova] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] No waiting events found dispatching network-vif-plugged-9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 703.227872] env[63273]: WARNING nova.compute.manager [req-c10238b8-ebe7-4677-90a3-2c5d4e4c78c0 req-3622d933-a072-4347-9fa1-8fe4919610b8 service nova] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Received unexpected event network-vif-plugged-9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0 for instance with vm_state building and task_state spawning. [ 703.332117] env[63273]: DEBUG nova.network.neutron [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Updating instance_info_cache with network_info: [{"id": "9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0", "address": "fa:16:3e:a8:32:b3", "network": {"id": "3176019f-6e7e-483a-bfa4-90972981063d", "bridge": "br-int", "label": "tempest-ImagesNegativeTestJSON-1295031639-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "70158b9660d14db684be2828e94f8e42", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c7c7f16b-a2e3-4d1f-9b7e-a44a9ebb589c", "external-id": "nsx-vlan-transportzone-772", "segmentation_id": 772, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9a2d73f2-bd", "ovs_interfaceid": "9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 703.346259] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Releasing lock "refresh_cache-f4d86853-25c8-4568-9b3f-8f07bcc5f068" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 703.346491] env[63273]: DEBUG nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Instance network_info: |[{"id": "9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0", "address": "fa:16:3e:a8:32:b3", "network": {"id": "3176019f-6e7e-483a-bfa4-90972981063d", "bridge": "br-int", "label": "tempest-ImagesNegativeTestJSON-1295031639-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "70158b9660d14db684be2828e94f8e42", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c7c7f16b-a2e3-4d1f-9b7e-a44a9ebb589c", "external-id": "nsx-vlan-transportzone-772", "segmentation_id": 772, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9a2d73f2-bd", "ovs_interfaceid": "9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 703.346894] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:a8:32:b3', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'c7c7f16b-a2e3-4d1f-9b7e-a44a9ebb589c', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 703.354916] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Creating folder: Project (70158b9660d14db684be2828e94f8e42). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 703.355617] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-24bb15df-c8fe-4455-9964-5e6fc7144d4b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.368062] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Created folder: Project (70158b9660d14db684be2828e94f8e42) in parent group-v986930. [ 703.368279] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Creating folder: Instances. Parent ref: group-v986971. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 703.368541] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-3dc2a165-cfb9-4f3b-b31d-03f90da6557f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.378421] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Created folder: Instances in parent group-v986971. [ 703.378666] env[63273]: DEBUG oslo.service.loopingcall [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 703.378851] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 703.379081] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-37a09db9-c360-4241-8111-77271461b3ea {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.399171] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 703.399171] env[63273]: value = "task-5072031" [ 703.399171] env[63273]: _type = "Task" [ 703.399171] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 703.410447] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072031, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 703.909177] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072031, 'name': CreateVM_Task, 'duration_secs': 0.305159} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 703.909557] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 703.910136] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 703.910332] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 703.910701] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 703.910962] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8068b10e-6032-4341-ab6a-a717a78752b2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.916334] env[63273]: DEBUG oslo_vmware.api [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Waiting for the task: (returnval){ [ 703.916334] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52c9317e-65c6-6992-e8e6-ad19135e7047" [ 703.916334] env[63273]: _type = "Task" [ 703.916334] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 703.924954] env[63273]: DEBUG oslo_vmware.api [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52c9317e-65c6-6992-e8e6-ad19135e7047, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 704.428345] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 704.428715] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 704.428945] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 705.330280] env[63273]: DEBUG nova.compute.manager [req-a7f8aac8-5728-4bdf-bd0b-ddd180e21e04 req-bcce6587-f699-4092-bbf9-114eff9eda83 service nova] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Received event network-changed-9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 705.330609] env[63273]: DEBUG nova.compute.manager [req-a7f8aac8-5728-4bdf-bd0b-ddd180e21e04 req-bcce6587-f699-4092-bbf9-114eff9eda83 service nova] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Refreshing instance network info cache due to event network-changed-9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 705.330834] env[63273]: DEBUG oslo_concurrency.lockutils [req-a7f8aac8-5728-4bdf-bd0b-ddd180e21e04 req-bcce6587-f699-4092-bbf9-114eff9eda83 service nova] Acquiring lock "refresh_cache-f4d86853-25c8-4568-9b3f-8f07bcc5f068" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 705.330834] env[63273]: DEBUG oslo_concurrency.lockutils [req-a7f8aac8-5728-4bdf-bd0b-ddd180e21e04 req-bcce6587-f699-4092-bbf9-114eff9eda83 service nova] Acquired lock "refresh_cache-f4d86853-25c8-4568-9b3f-8f07bcc5f068" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 705.331050] env[63273]: DEBUG nova.network.neutron [req-a7f8aac8-5728-4bdf-bd0b-ddd180e21e04 req-bcce6587-f699-4092-bbf9-114eff9eda83 service nova] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Refreshing network info cache for port 9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 705.727370] env[63273]: DEBUG nova.network.neutron [req-a7f8aac8-5728-4bdf-bd0b-ddd180e21e04 req-bcce6587-f699-4092-bbf9-114eff9eda83 service nova] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Updated VIF entry in instance network info cache for port 9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 705.727708] env[63273]: DEBUG nova.network.neutron [req-a7f8aac8-5728-4bdf-bd0b-ddd180e21e04 req-bcce6587-f699-4092-bbf9-114eff9eda83 service nova] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Updating instance_info_cache with network_info: [{"id": "9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0", "address": "fa:16:3e:a8:32:b3", "network": {"id": "3176019f-6e7e-483a-bfa4-90972981063d", "bridge": "br-int", "label": "tempest-ImagesNegativeTestJSON-1295031639-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "70158b9660d14db684be2828e94f8e42", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c7c7f16b-a2e3-4d1f-9b7e-a44a9ebb589c", "external-id": "nsx-vlan-transportzone-772", "segmentation_id": 772, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9a2d73f2-bd", "ovs_interfaceid": "9a2d73f2-bd8e-4cbc-a69c-56863fdff8f0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 705.737409] env[63273]: DEBUG oslo_concurrency.lockutils [req-a7f8aac8-5728-4bdf-bd0b-ddd180e21e04 req-bcce6587-f699-4092-bbf9-114eff9eda83 service nova] Releasing lock "refresh_cache-f4d86853-25c8-4568-9b3f-8f07bcc5f068" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 740.893299] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 740.893623] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 740.908210] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 740.908422] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 740.908591] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 740.908752] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 740.909921] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ae5a16a-cea4-43c5-b50e-1de3036e1415 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.920077] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed79b328-e5b8-4b73-bd84-bbf4a886c6fb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.935991] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbc42307-f8af-4add-bf5a-c3ddcc85d273 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.943999] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a7b98be-300c-4776-8f11-8533e294f4e0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.975020] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180526MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 740.975206] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 740.975411] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 741.071105] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 6699f6e7-203d-4d8a-bc9e-8c8d5427db36 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.082408] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4846a37e-b175-4f23-b8c1-34a6b124513d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.092662] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 811bed85-6a4d-4340-9e4f-a2acd86d921c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.104218] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance cc6f5d6c-1f3c-4ead-8ee4-9b59923ac57b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.104218] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77493b32-817c-4be8-a42d-8d48e707d41f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 741.114879] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 362b7115-a270-407f-a53c-506c863c8fe1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.125215] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 68fe9a49-e561-45da-afcb-e804e2109e76 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.135791] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8728833b-c3ac-49b2-9a33-ecfae9597e9e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.145922] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 02e9cc93-bdc3-4220-a22e-e29e959a5377 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.158769] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 451c95b6-a407-4911-bc40-83f2e2e7a75f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.169727] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8dd5ad91-d09a-41dd-a8d0-fd22e96fc279 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.182047] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c936b733-7faa-4e73-99c5-2ac11d5421ff has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.182047] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f1dddee3-33d6-4f55-bd91-9ce2fe6be72b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 741.193052] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance ff6cf180-c0fe-4b4f-9bac-08e9e3dc7c1d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.193052] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 741.193052] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance ab9cf87a-b74f-4b2f-842d-26e16b7fdb92 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 741.193251] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 94d19d7f-8eb7-4c93-aff4-3b1404338240 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 741.204104] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7d000257-5fc1-43fa-9fc5-a07519a6feea has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.220602] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.220788] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 741.220940] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 02ed8dca-f6ae-47b0-a047-f89f858f5ff2 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 741.232689] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f889087d-5f22-4de6-8dff-819d2afb7240 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.244428] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 3af8fe9a-643b-4650-a036-6f1b9c3c8d03 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.255691] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.268707] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1979e66a-172d-4130-90ea-9f0034b13071 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.279530] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 6b48c313-e3d5-4d3c-83c5-3c12d948abba has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.291142] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 11838596-1730-4c23-bbf2-3159d3c6d72d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.302422] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 707e02eb-1471-4ddd-be9e-a41f33d42482 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.302641] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 11a32984-5548-4db4-8788-9b221ba381df actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 741.312909] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a4e56801-a8b8-448f-9b1a-d228ceeaaa46 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.323371] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 717fd312-ac1d-42b6-9f93-13b9f235bd23 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.323553] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5501c192-867b-40c0-9bea-c3c44865d2f4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 741.334625] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e36789b5-f814-4105-b144-361fef9e0d0e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.345862] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a12ff1bf-8cfa-4d03-828d-6b34727f61bf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.346032] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a5c37718-0067-4f5e-ba7e-533f13529739 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 741.356865] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 741.357138] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 741.357305] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '3', 'num_instances': '10', 'num_vm_building': '10', 'num_task_spawning': '10', 'num_os_type_None': '10', 'num_proj_726f39967ec7455dadc4ed498eff094b': '1', 'io_workload': '10', 'num_proj_37d368bc456740739821f51e3a25f2fb': '1', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'num_proj_93e7b65b07c44f5c88154e6f398bdbd8': '1', 'num_proj_d806b4c5831b457fa075e387f55d8ad5': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_4b844f0a784d4f20a5adbf7354c75cc7': '1', 'num_proj_ad03061e544f4b17abb83d9596c461a0': '1', 'num_proj_06e0618687eb4144b9aacdc90a42879d': '1', 'num_proj_70158b9660d14db684be2828e94f8e42': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 741.824367] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca1eff19-2045-4e26-9575-c3a38ced6da1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 741.832589] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff1ae63e-7681-4ae6-9cd7-9e8c9e51ec00 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 741.863917] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e171d9f8-7bc4-4041-9330-2f4ae813115e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 741.872878] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f04d23e4-d87b-4357-87b6-2475bc484d8d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 741.886921] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 741.895472] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 741.914236] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 741.914421] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.939s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 742.912899] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 742.913243] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 742.913308] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 742.913453] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 743.887356] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.891015] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 744.886770] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 744.908505] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 744.908650] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 744.908770] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 744.929571] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 744.929982] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 744.929982] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 744.929982] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 744.930144] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 744.930250] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 744.930370] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 744.930589] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 744.930747] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 744.930871] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 744.930993] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 744.931469] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 745.053910] env[63273]: WARNING oslo_vmware.rw_handles [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 745.053910] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 745.053910] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 745.053910] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 745.053910] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 745.053910] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 745.053910] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 745.053910] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 745.053910] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 745.053910] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 745.053910] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 745.053910] env[63273]: ERROR oslo_vmware.rw_handles [ 745.054333] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/054a93d6-af3b-4ee1-beb7-23bd6478ba61/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 745.056118] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 745.056307] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Copying Virtual Disk [datastore1] vmware_temp/054a93d6-af3b-4ee1-beb7-23bd6478ba61/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/054a93d6-af3b-4ee1-beb7-23bd6478ba61/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 745.056593] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-244cccd9-a3ee-4abe-894b-46886399ed3d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.065280] env[63273]: DEBUG oslo_vmware.api [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Waiting for the task: (returnval){ [ 745.065280] env[63273]: value = "task-5072032" [ 745.065280] env[63273]: _type = "Task" [ 745.065280] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 745.074352] env[63273]: DEBUG oslo_vmware.api [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Task: {'id': task-5072032, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 745.529138] env[63273]: DEBUG oslo_concurrency.lockutils [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquiring lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 745.575944] env[63273]: DEBUG oslo_vmware.exceptions [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 745.576240] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 745.576839] env[63273]: ERROR nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 745.576839] env[63273]: Faults: ['InvalidArgument'] [ 745.576839] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] Traceback (most recent call last): [ 745.576839] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 745.576839] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] yield resources [ 745.576839] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 745.576839] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] self.driver.spawn(context, instance, image_meta, [ 745.576839] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 745.576839] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] self._vmops.spawn(context, instance, image_meta, injected_files, [ 745.576839] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 745.576839] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] self._fetch_image_if_missing(context, vi) [ 745.576839] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] image_cache(vi, tmp_image_ds_loc) [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] vm_util.copy_virtual_disk( [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] session._wait_for_task(vmdk_copy_task) [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] return self.wait_for_task(task_ref) [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] return evt.wait() [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] result = hub.switch() [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 745.577215] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] return self.greenlet.switch() [ 745.577584] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 745.577584] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] self.f(*self.args, **self.kw) [ 745.577584] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 745.577584] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] raise exceptions.translate_fault(task_info.error) [ 745.577584] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 745.577584] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] Faults: ['InvalidArgument'] [ 745.577584] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] [ 745.577584] env[63273]: INFO nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Terminating instance [ 745.578771] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 745.578981] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 745.579238] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ea1fe011-a0fb-4430-8f05-ab81b5d31bcb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.581623] env[63273]: DEBUG nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 745.581820] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 745.582673] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67ef267c-a55e-4286-a3b2-3d944563d716 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.590503] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 745.590734] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-f6f043c2-97a5-4462-8d44-794a6bb0f4fb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.593381] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 745.593563] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 745.594644] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d7365c65-8a0e-4984-a049-b066f430e65e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.600299] env[63273]: DEBUG oslo_vmware.api [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Waiting for the task: (returnval){ [ 745.600299] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52486d79-463b-cee2-4594-86017b440588" [ 745.600299] env[63273]: _type = "Task" [ 745.600299] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 745.612093] env[63273]: DEBUG oslo_vmware.api [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52486d79-463b-cee2-4594-86017b440588, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 745.666068] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 745.666310] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 745.666378] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Deleting the datastore file [datastore1] 11a32984-5548-4db4-8788-9b221ba381df {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 745.666870] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-29f9e8f7-070c-4f4e-b8e3-fe0180c4323b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.674125] env[63273]: DEBUG oslo_vmware.api [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Waiting for the task: (returnval){ [ 745.674125] env[63273]: value = "task-5072034" [ 745.674125] env[63273]: _type = "Task" [ 745.674125] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 745.682982] env[63273]: DEBUG oslo_vmware.api [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Task: {'id': task-5072034, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 746.112081] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 746.112383] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Creating directory with path [datastore1] vmware_temp/f1da1657-63a6-458d-af82-fa02a3a62ea0/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 746.112935] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0255b7e3-f9dd-43af-9574-c816337feb53 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.125622] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Created directory with path [datastore1] vmware_temp/f1da1657-63a6-458d-af82-fa02a3a62ea0/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 746.125622] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Fetch image to [datastore1] vmware_temp/f1da1657-63a6-458d-af82-fa02a3a62ea0/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 746.125777] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/f1da1657-63a6-458d-af82-fa02a3a62ea0/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 746.126505] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08cc7afc-022e-40c9-8462-5e47495b0757 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.133950] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f31f7b6-a3f4-4054-a713-d3896f91c43d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.143450] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e106e1e-4757-4232-bd33-e572e731f547 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.175066] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebd97d9b-8328-492c-8167-7542b9385315 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.187330] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-646ace24-1fcc-463f-a350-a5109c10afc5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.189138] env[63273]: DEBUG oslo_vmware.api [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Task: {'id': task-5072034, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.073693} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 746.189392] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 746.189572] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 746.189739] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 746.189907] env[63273]: INFO nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Took 0.61 seconds to destroy the instance on the hypervisor. [ 746.192179] env[63273]: DEBUG nova.compute.claims [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 746.192369] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.192587] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.287891] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 746.352610] env[63273]: DEBUG oslo_vmware.rw_handles [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f1da1657-63a6-458d-af82-fa02a3a62ea0/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 746.413843] env[63273]: DEBUG oslo_vmware.rw_handles [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 746.414205] env[63273]: DEBUG oslo_vmware.rw_handles [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f1da1657-63a6-458d-af82-fa02a3a62ea0/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 746.767934] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1835f517-9423-4f8f-89fb-df48e7b339d5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.776560] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b335a963-c576-400e-a3b6-0afc8e39e616 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.808969] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5016076-0e5a-44fb-b452-4cd19366a6bf {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.817146] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80d3498f-7da7-4bb2-b341-937f3406938a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.830918] env[63273]: DEBUG nova.compute.provider_tree [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 746.843987] env[63273]: DEBUG nova.scheduler.client.report [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 746.862131] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.669s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 746.862709] env[63273]: ERROR nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 746.862709] env[63273]: Faults: ['InvalidArgument'] [ 746.862709] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] Traceback (most recent call last): [ 746.862709] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 746.862709] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] self.driver.spawn(context, instance, image_meta, [ 746.862709] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 746.862709] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] self._vmops.spawn(context, instance, image_meta, injected_files, [ 746.862709] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 746.862709] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] self._fetch_image_if_missing(context, vi) [ 746.862709] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 746.862709] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] image_cache(vi, tmp_image_ds_loc) [ 746.862709] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] vm_util.copy_virtual_disk( [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] session._wait_for_task(vmdk_copy_task) [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] return self.wait_for_task(task_ref) [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] return evt.wait() [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] result = hub.switch() [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] return self.greenlet.switch() [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 746.863096] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] self.f(*self.args, **self.kw) [ 746.863439] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 746.863439] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] raise exceptions.translate_fault(task_info.error) [ 746.863439] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 746.863439] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] Faults: ['InvalidArgument'] [ 746.863439] env[63273]: ERROR nova.compute.manager [instance: 11a32984-5548-4db4-8788-9b221ba381df] [ 746.863439] env[63273]: DEBUG nova.compute.utils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 746.864968] env[63273]: DEBUG nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Build of instance 11a32984-5548-4db4-8788-9b221ba381df was re-scheduled: A specified parameter was not correct: fileType [ 746.864968] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 746.865357] env[63273]: DEBUG nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 746.865574] env[63273]: DEBUG nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 746.865706] env[63273]: DEBUG nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 746.865868] env[63273]: DEBUG nova.network.neutron [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 747.309910] env[63273]: DEBUG nova.network.neutron [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 747.327713] env[63273]: INFO nova.compute.manager [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] [instance: 11a32984-5548-4db4-8788-9b221ba381df] Took 0.46 seconds to deallocate network for instance. [ 747.449044] env[63273]: INFO nova.scheduler.client.report [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Deleted allocations for instance 11a32984-5548-4db4-8788-9b221ba381df [ 747.473147] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f4312887-9e3e-4097-a24e-a49182ed2bf8 tempest-ImagesOneServerNegativeTestJSON-1182666320 tempest-ImagesOneServerNegativeTestJSON-1182666320-project-member] Lock "11a32984-5548-4db4-8788-9b221ba381df" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 193.576s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 747.474572] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "11a32984-5548-4db4-8788-9b221ba381df" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 183.198s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 747.474770] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 11a32984-5548-4db4-8788-9b221ba381df] During sync_power_state the instance has a pending task (spawning). Skip. [ 747.475289] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "11a32984-5548-4db4-8788-9b221ba381df" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 747.495987] env[63273]: DEBUG nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 747.562204] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 747.562468] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 747.564203] env[63273]: INFO nova.compute.claims [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 748.071947] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff5b7ab7-301a-4201-8353-1ba36dce9e7a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.080605] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f91a4a7e-de1e-4ada-b943-c8bf40129861 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.111413] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9879a90f-cea9-40df-bc99-5415c22b932f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.119787] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3dcd8fa-e5cc-4c4f-b57d-03fcec8cc7dc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.133912] env[63273]: DEBUG nova.compute.provider_tree [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 748.142754] env[63273]: DEBUG nova.scheduler.client.report [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 748.162648] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.599s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 748.162648] env[63273]: DEBUG nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 748.199126] env[63273]: DEBUG nova.compute.utils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 748.200457] env[63273]: DEBUG nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 748.200633] env[63273]: DEBUG nova.network.neutron [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 748.213980] env[63273]: DEBUG nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 748.321244] env[63273]: DEBUG nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 748.329717] env[63273]: DEBUG nova.policy [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '33f3e3b867bd4d508863ce0366cbddf9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f1ef99990442451e927a36cdbb924a86', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 748.366141] env[63273]: DEBUG nova.virt.hardware [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 748.366141] env[63273]: DEBUG nova.virt.hardware [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 748.366141] env[63273]: DEBUG nova.virt.hardware [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 748.366340] env[63273]: DEBUG nova.virt.hardware [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 748.366340] env[63273]: DEBUG nova.virt.hardware [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 748.366340] env[63273]: DEBUG nova.virt.hardware [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 748.366340] env[63273]: DEBUG nova.virt.hardware [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 748.366461] env[63273]: DEBUG nova.virt.hardware [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 748.366684] env[63273]: DEBUG nova.virt.hardware [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 748.366795] env[63273]: DEBUG nova.virt.hardware [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 748.366973] env[63273]: DEBUG nova.virt.hardware [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 748.367951] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-635a8e0f-cffa-4938-b504-2223bc31c79f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.377233] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8a825ed-eaeb-4175-9610-452c682b2ef0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.767271] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquiring lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 749.143153] env[63273]: DEBUG nova.network.neutron [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Successfully created port: bc1cce49-6568-44e3-811f-5f34ae13d67f {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 750.648079] env[63273]: DEBUG nova.network.neutron [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Successfully updated port: bc1cce49-6568-44e3-811f-5f34ae13d67f {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 750.669764] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquiring lock "refresh_cache-4ac5c733-988f-4428-ad6f-134d9f174e45" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 750.669913] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquired lock "refresh_cache-4ac5c733-988f-4428-ad6f-134d9f174e45" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 750.670076] env[63273]: DEBUG nova.network.neutron [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 750.730251] env[63273]: DEBUG nova.network.neutron [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 750.866106] env[63273]: DEBUG nova.compute.manager [req-9a6f13a9-4dc0-4b59-953c-d383ac0dd4d8 req-5e2aada6-5932-409d-9892-6039628fae51 service nova] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Received event network-vif-plugged-bc1cce49-6568-44e3-811f-5f34ae13d67f {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 750.866369] env[63273]: DEBUG oslo_concurrency.lockutils [req-9a6f13a9-4dc0-4b59-953c-d383ac0dd4d8 req-5e2aada6-5932-409d-9892-6039628fae51 service nova] Acquiring lock "4ac5c733-988f-4428-ad6f-134d9f174e45-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 750.866647] env[63273]: DEBUG oslo_concurrency.lockutils [req-9a6f13a9-4dc0-4b59-953c-d383ac0dd4d8 req-5e2aada6-5932-409d-9892-6039628fae51 service nova] Lock "4ac5c733-988f-4428-ad6f-134d9f174e45-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 750.866757] env[63273]: DEBUG oslo_concurrency.lockutils [req-9a6f13a9-4dc0-4b59-953c-d383ac0dd4d8 req-5e2aada6-5932-409d-9892-6039628fae51 service nova] Lock "4ac5c733-988f-4428-ad6f-134d9f174e45-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 750.866883] env[63273]: DEBUG nova.compute.manager [req-9a6f13a9-4dc0-4b59-953c-d383ac0dd4d8 req-5e2aada6-5932-409d-9892-6039628fae51 service nova] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] No waiting events found dispatching network-vif-plugged-bc1cce49-6568-44e3-811f-5f34ae13d67f {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 750.867102] env[63273]: WARNING nova.compute.manager [req-9a6f13a9-4dc0-4b59-953c-d383ac0dd4d8 req-5e2aada6-5932-409d-9892-6039628fae51 service nova] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Received unexpected event network-vif-plugged-bc1cce49-6568-44e3-811f-5f34ae13d67f for instance with vm_state building and task_state spawning. [ 750.983413] env[63273]: DEBUG nova.network.neutron [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Updating instance_info_cache with network_info: [{"id": "bc1cce49-6568-44e3-811f-5f34ae13d67f", "address": "fa:16:3e:a0:ae:6c", "network": {"id": "0d96cecb-bb61-457d-a211-dd6e66629d3b", "bridge": "br-int", "label": "tempest-ServersTestJSON-1627750150-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f1ef99990442451e927a36cdbb924a86", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "fd8b2b4e-f09d-4af6-9759-d372870e9b5f", "external-id": "nsx-vlan-transportzone-800", "segmentation_id": 800, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbc1cce49-65", "ovs_interfaceid": "bc1cce49-6568-44e3-811f-5f34ae13d67f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 751.000486] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Releasing lock "refresh_cache-4ac5c733-988f-4428-ad6f-134d9f174e45" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 751.002288] env[63273]: DEBUG nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Instance network_info: |[{"id": "bc1cce49-6568-44e3-811f-5f34ae13d67f", "address": "fa:16:3e:a0:ae:6c", "network": {"id": "0d96cecb-bb61-457d-a211-dd6e66629d3b", "bridge": "br-int", "label": "tempest-ServersTestJSON-1627750150-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f1ef99990442451e927a36cdbb924a86", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "fd8b2b4e-f09d-4af6-9759-d372870e9b5f", "external-id": "nsx-vlan-transportzone-800", "segmentation_id": 800, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbc1cce49-65", "ovs_interfaceid": "bc1cce49-6568-44e3-811f-5f34ae13d67f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 751.002436] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:a0:ae:6c', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'fd8b2b4e-f09d-4af6-9759-d372870e9b5f', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'bc1cce49-6568-44e3-811f-5f34ae13d67f', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 751.010309] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Creating folder: Project (f1ef99990442451e927a36cdbb924a86). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 751.011447] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-219ca857-f839-4876-981c-6732857e316a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.025023] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Created folder: Project (f1ef99990442451e927a36cdbb924a86) in parent group-v986930. [ 751.025023] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Creating folder: Instances. Parent ref: group-v986974. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 751.025023] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-1b102a72-99af-4616-b904-9a5b4f91a604 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.035316] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Created folder: Instances in parent group-v986974. [ 751.035843] env[63273]: DEBUG oslo.service.loopingcall [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 751.036162] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 751.036539] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-5507fb1a-946b-4f66-8122-ad3859a3d445 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.060019] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 751.060019] env[63273]: value = "task-5072037" [ 751.060019] env[63273]: _type = "Task" [ 751.060019] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 751.066658] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072037, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 751.568568] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072037, 'name': CreateVM_Task, 'duration_secs': 0.316422} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 751.568786] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 751.569887] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 751.570415] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 751.570761] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 751.571044] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-dca57aad-a493-4b3e-8422-801e654abc4c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.576722] env[63273]: DEBUG oslo_vmware.api [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Waiting for the task: (returnval){ [ 751.576722] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5224b2e9-69d8-8ee6-00bf-ab870603f021" [ 751.576722] env[63273]: _type = "Task" [ 751.576722] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 751.585927] env[63273]: DEBUG oslo_vmware.api [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5224b2e9-69d8-8ee6-00bf-ab870603f021, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 752.096759] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 752.096759] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 752.096759] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 753.034953] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 753.048985] env[63273]: DEBUG nova.compute.manager [req-c34a50e3-c39d-4d8d-addb-36023bf3b5d1 req-5d17b4d9-e2de-4ecf-bab0-9e964d34ae8e service nova] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Received event network-changed-bc1cce49-6568-44e3-811f-5f34ae13d67f {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 753.049202] env[63273]: DEBUG nova.compute.manager [req-c34a50e3-c39d-4d8d-addb-36023bf3b5d1 req-5d17b4d9-e2de-4ecf-bab0-9e964d34ae8e service nova] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Refreshing instance network info cache due to event network-changed-bc1cce49-6568-44e3-811f-5f34ae13d67f. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 753.049416] env[63273]: DEBUG oslo_concurrency.lockutils [req-c34a50e3-c39d-4d8d-addb-36023bf3b5d1 req-5d17b4d9-e2de-4ecf-bab0-9e964d34ae8e service nova] Acquiring lock "refresh_cache-4ac5c733-988f-4428-ad6f-134d9f174e45" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 753.049558] env[63273]: DEBUG oslo_concurrency.lockutils [req-c34a50e3-c39d-4d8d-addb-36023bf3b5d1 req-5d17b4d9-e2de-4ecf-bab0-9e964d34ae8e service nova] Acquired lock "refresh_cache-4ac5c733-988f-4428-ad6f-134d9f174e45" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 753.049717] env[63273]: DEBUG nova.network.neutron [req-c34a50e3-c39d-4d8d-addb-36023bf3b5d1 req-5d17b4d9-e2de-4ecf-bab0-9e964d34ae8e service nova] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Refreshing network info cache for port bc1cce49-6568-44e3-811f-5f34ae13d67f {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 753.561248] env[63273]: DEBUG nova.network.neutron [req-c34a50e3-c39d-4d8d-addb-36023bf3b5d1 req-5d17b4d9-e2de-4ecf-bab0-9e964d34ae8e service nova] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Updated VIF entry in instance network info cache for port bc1cce49-6568-44e3-811f-5f34ae13d67f. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 753.561619] env[63273]: DEBUG nova.network.neutron [req-c34a50e3-c39d-4d8d-addb-36023bf3b5d1 req-5d17b4d9-e2de-4ecf-bab0-9e964d34ae8e service nova] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Updating instance_info_cache with network_info: [{"id": "bc1cce49-6568-44e3-811f-5f34ae13d67f", "address": "fa:16:3e:a0:ae:6c", "network": {"id": "0d96cecb-bb61-457d-a211-dd6e66629d3b", "bridge": "br-int", "label": "tempest-ServersTestJSON-1627750150-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f1ef99990442451e927a36cdbb924a86", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "fd8b2b4e-f09d-4af6-9759-d372870e9b5f", "external-id": "nsx-vlan-transportzone-800", "segmentation_id": 800, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbc1cce49-65", "ovs_interfaceid": "bc1cce49-6568-44e3-811f-5f34ae13d67f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 753.573960] env[63273]: DEBUG oslo_concurrency.lockutils [req-c34a50e3-c39d-4d8d-addb-36023bf3b5d1 req-5d17b4d9-e2de-4ecf-bab0-9e964d34ae8e service nova] Releasing lock "refresh_cache-4ac5c733-988f-4428-ad6f-134d9f174e45" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 758.897892] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "13511c60-50cd-44ed-969e-c5fc29b0125e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 758.898400] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 759.658947] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquiring lock "a5c37718-0067-4f5e-ba7e-533f13529739" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 760.811058] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 762.419500] env[63273]: DEBUG oslo_concurrency.lockutils [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquiring lock "94d19d7f-8eb7-4c93-aff4-3b1404338240" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 762.947606] env[63273]: DEBUG oslo_concurrency.lockutils [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquiring lock "5501c192-867b-40c0-9bea-c3c44865d2f4" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 766.128084] env[63273]: DEBUG oslo_concurrency.lockutils [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquiring lock "77493b32-817c-4be8-a42d-8d48e707d41f" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 766.587488] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquiring lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 766.896625] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquiring lock "4ac5c733-988f-4428-ad6f-134d9f174e45" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.661624] env[63273]: DEBUG oslo_concurrency.lockutils [None req-32435d1f-c79a-481f-a3ae-3f3f54739371 tempest-ServersTestFqdnHostnames-1404605347 tempest-ServersTestFqdnHostnames-1404605347-project-member] Acquiring lock "2eb7836e-f1ce-4eea-9cee-f9075cdf6d99" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.661947] env[63273]: DEBUG oslo_concurrency.lockutils [None req-32435d1f-c79a-481f-a3ae-3f3f54739371 tempest-ServersTestFqdnHostnames-1404605347 tempest-ServersTestFqdnHostnames-1404605347-project-member] Lock "2eb7836e-f1ce-4eea-9cee-f9075cdf6d99" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 788.714364] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7fb717e5-dff6-4664-80ce-1dd824c73301 tempest-ServerAddressesNegativeTestJSON-375425406 tempest-ServerAddressesNegativeTestJSON-375425406-project-member] Acquiring lock "3e73331f-ce47-4310-98dc-b1a9ac168d30" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 788.715372] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7fb717e5-dff6-4664-80ce-1dd824c73301 tempest-ServerAddressesNegativeTestJSON-375425406 tempest-ServerAddressesNegativeTestJSON-375425406-project-member] Lock "3e73331f-ce47-4310-98dc-b1a9ac168d30" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 795.789123] env[63273]: WARNING oslo_vmware.rw_handles [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 795.789123] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 795.789123] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 795.789123] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 795.789123] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 795.789123] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 795.789123] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 795.789123] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 795.789123] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 795.789123] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 795.789123] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 795.789123] env[63273]: ERROR oslo_vmware.rw_handles [ 795.789810] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/f1da1657-63a6-458d-af82-fa02a3a62ea0/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 795.794619] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 795.794619] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Copying Virtual Disk [datastore1] vmware_temp/f1da1657-63a6-458d-af82-fa02a3a62ea0/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/f1da1657-63a6-458d-af82-fa02a3a62ea0/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 795.794619] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-c1b0579f-9e36-4b72-8931-788acd2aad25 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.801379] env[63273]: DEBUG oslo_vmware.api [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Waiting for the task: (returnval){ [ 795.801379] env[63273]: value = "task-5072038" [ 795.801379] env[63273]: _type = "Task" [ 795.801379] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 795.816513] env[63273]: DEBUG oslo_vmware.api [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Task: {'id': task-5072038, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 796.317669] env[63273]: DEBUG oslo_vmware.exceptions [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 796.317669] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 796.318148] env[63273]: ERROR nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 796.318148] env[63273]: Faults: ['InvalidArgument'] [ 796.318148] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Traceback (most recent call last): [ 796.318148] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 796.318148] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] yield resources [ 796.318148] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 796.318148] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] self.driver.spawn(context, instance, image_meta, [ 796.318148] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 796.318148] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 796.318148] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 796.318148] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] self._fetch_image_if_missing(context, vi) [ 796.318148] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] image_cache(vi, tmp_image_ds_loc) [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] vm_util.copy_virtual_disk( [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] session._wait_for_task(vmdk_copy_task) [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] return self.wait_for_task(task_ref) [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] return evt.wait() [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] result = hub.switch() [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 796.318565] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] return self.greenlet.switch() [ 796.318950] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 796.318950] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] self.f(*self.args, **self.kw) [ 796.318950] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 796.318950] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] raise exceptions.translate_fault(task_info.error) [ 796.318950] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 796.318950] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Faults: ['InvalidArgument'] [ 796.318950] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] [ 796.318950] env[63273]: INFO nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Terminating instance [ 796.322866] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 796.323113] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 796.326697] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e4ad8e47-03ad-4d4f-b943-28960fcf28ea {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.329863] env[63273]: DEBUG nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 796.330266] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 796.331083] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2829145-5331-4c07-a1ba-9d50859c2783 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.339537] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 796.339746] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-724ffd4b-e1df-4856-ad49-8f7aebf2e1a1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.342634] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 796.343447] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 796.344581] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a7c0833a-1bcf-4d34-8dfd-0b95f47a9a3f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.354179] env[63273]: DEBUG oslo_vmware.api [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Waiting for the task: (returnval){ [ 796.354179] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52555759-5457-0d0d-43c7-9dff4ac744c2" [ 796.354179] env[63273]: _type = "Task" [ 796.354179] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 796.368327] env[63273]: DEBUG oslo_vmware.api [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52555759-5457-0d0d-43c7-9dff4ac744c2, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 796.421077] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 796.421311] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 796.422596] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Deleting the datastore file [datastore1] 02ed8dca-f6ae-47b0-a047-f89f858f5ff2 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 796.422769] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-441bca7d-36fa-429b-8f36-1368659d80d0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.434256] env[63273]: DEBUG oslo_vmware.api [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Waiting for the task: (returnval){ [ 796.434256] env[63273]: value = "task-5072040" [ 796.434256] env[63273]: _type = "Task" [ 796.434256] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 796.445237] env[63273]: DEBUG oslo_vmware.api [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Task: {'id': task-5072040, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 796.516896] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4cbe97e-784e-4d13-9107-28eb1982fa47 tempest-InstanceActionsV221TestJSON-243712533 tempest-InstanceActionsV221TestJSON-243712533-project-member] Acquiring lock "f5edcebb-61c8-47f5-a925-fd3f0b7a1389" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.517195] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4cbe97e-784e-4d13-9107-28eb1982fa47 tempest-InstanceActionsV221TestJSON-243712533 tempest-InstanceActionsV221TestJSON-243712533-project-member] Lock "f5edcebb-61c8-47f5-a925-fd3f0b7a1389" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.864847] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 796.865138] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Creating directory with path [datastore1] vmware_temp/fadf268e-8dc2-4b2b-9cbf-2d70f83a3cd2/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 796.865488] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-cf4e9e42-b2ab-49a4-9da9-30da1792750c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.880420] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Created directory with path [datastore1] vmware_temp/fadf268e-8dc2-4b2b-9cbf-2d70f83a3cd2/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 796.880815] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Fetch image to [datastore1] vmware_temp/fadf268e-8dc2-4b2b-9cbf-2d70f83a3cd2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 796.881225] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/fadf268e-8dc2-4b2b-9cbf-2d70f83a3cd2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 796.882804] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab3abc99-24e8-4ff2-a42f-1c037b6b3bba {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.898951] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1be9184a-9296-4233-92fe-4c4380f5ad76 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.913828] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76313971-a78d-4a77-bd46-d3a258dd9b6e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.952458] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6760a5a-a070-40ff-8f1e-4a52ebca2d44 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.963095] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-020eda60-31d5-46de-a8ea-4b6527e25ccc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.964762] env[63273]: DEBUG oslo_vmware.api [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Task: {'id': task-5072040, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.086039} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 796.965025] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 796.965208] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 796.965376] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 796.965543] env[63273]: INFO nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Took 0.64 seconds to destroy the instance on the hypervisor. [ 796.968283] env[63273]: DEBUG nova.compute.claims [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 796.968454] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.970501] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.992716] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 797.072137] env[63273]: DEBUG oslo_vmware.rw_handles [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/fadf268e-8dc2-4b2b-9cbf-2d70f83a3cd2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 797.140382] env[63273]: DEBUG oslo_vmware.rw_handles [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 797.140569] env[63273]: DEBUG oslo_vmware.rw_handles [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/fadf268e-8dc2-4b2b-9cbf-2d70f83a3cd2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 797.554170] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73d346f3-2d81-416d-bf83-f8de63e3a781 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.562743] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1a949f1-4f68-4cbb-9802-e1331ef8c749 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.597024] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f1af379-2e4f-47dc-9754-1482831b9d05 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.607020] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86259e7d-f6a5-4b63-880d-0487729223a1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.621451] env[63273]: DEBUG nova.compute.provider_tree [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 797.631197] env[63273]: DEBUG nova.scheduler.client.report [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 797.649614] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.681s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 797.650225] env[63273]: ERROR nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 797.650225] env[63273]: Faults: ['InvalidArgument'] [ 797.650225] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Traceback (most recent call last): [ 797.650225] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 797.650225] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] self.driver.spawn(context, instance, image_meta, [ 797.650225] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 797.650225] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 797.650225] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 797.650225] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] self._fetch_image_if_missing(context, vi) [ 797.650225] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 797.650225] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] image_cache(vi, tmp_image_ds_loc) [ 797.650225] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] vm_util.copy_virtual_disk( [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] session._wait_for_task(vmdk_copy_task) [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] return self.wait_for_task(task_ref) [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] return evt.wait() [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] result = hub.switch() [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] return self.greenlet.switch() [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 797.650572] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] self.f(*self.args, **self.kw) [ 797.650895] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 797.650895] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] raise exceptions.translate_fault(task_info.error) [ 797.650895] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 797.650895] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Faults: ['InvalidArgument'] [ 797.650895] env[63273]: ERROR nova.compute.manager [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] [ 797.651103] env[63273]: DEBUG nova.compute.utils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 797.654145] env[63273]: DEBUG nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Build of instance 02ed8dca-f6ae-47b0-a047-f89f858f5ff2 was re-scheduled: A specified parameter was not correct: fileType [ 797.654145] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 797.654559] env[63273]: DEBUG nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 797.654735] env[63273]: DEBUG nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 797.654889] env[63273]: DEBUG nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 797.655081] env[63273]: DEBUG nova.network.neutron [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 798.443636] env[63273]: DEBUG nova.network.neutron [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 798.459115] env[63273]: INFO nova.compute.manager [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Took 0.80 seconds to deallocate network for instance. [ 798.638723] env[63273]: INFO nova.scheduler.client.report [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Deleted allocations for instance 02ed8dca-f6ae-47b0-a047-f89f858f5ff2 [ 798.667087] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5661ef63-d91d-4c3b-8161-ae419da5f486 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 250.554s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 798.668269] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 234.393s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 798.668462] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] During sync_power_state the instance has a pending task (spawning). Skip. [ 798.668637] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 798.673022] env[63273]: DEBUG oslo_concurrency.lockutils [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 53.141s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 798.673022] env[63273]: DEBUG oslo_concurrency.lockutils [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Acquiring lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 798.673022] env[63273]: DEBUG oslo_concurrency.lockutils [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 798.673280] env[63273]: DEBUG oslo_concurrency.lockutils [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 798.673598] env[63273]: INFO nova.compute.manager [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Terminating instance [ 798.677998] env[63273]: DEBUG nova.compute.manager [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 798.678313] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 798.678968] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c9f26c7d-438f-4fe7-b88b-2591a8ee7812 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.691232] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5bc04c7-7bfc-4b52-9e0c-204753f8eea0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.704044] env[63273]: DEBUG nova.compute.manager [None req-8cd7758d-9e5f-463b-9166-d44861ef7b72 tempest-AttachInterfacesV270Test-389945933 tempest-AttachInterfacesV270Test-389945933-project-member] [instance: 3af8fe9a-643b-4650-a036-6f1b9c3c8d03] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 798.733634] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 02ed8dca-f6ae-47b0-a047-f89f858f5ff2 could not be found. [ 798.733849] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 798.734061] env[63273]: INFO nova.compute.manager [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Took 0.06 seconds to destroy the instance on the hypervisor. [ 798.734334] env[63273]: DEBUG oslo.service.loopingcall [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 798.734568] env[63273]: DEBUG nova.compute.manager [-] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 798.734667] env[63273]: DEBUG nova.network.neutron [-] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 798.737339] env[63273]: DEBUG nova.compute.manager [None req-8cd7758d-9e5f-463b-9166-d44861ef7b72 tempest-AttachInterfacesV270Test-389945933 tempest-AttachInterfacesV270Test-389945933-project-member] [instance: 3af8fe9a-643b-4650-a036-6f1b9c3c8d03] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 798.770456] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8cd7758d-9e5f-463b-9166-d44861ef7b72 tempest-AttachInterfacesV270Test-389945933 tempest-AttachInterfacesV270Test-389945933-project-member] Lock "3af8fe9a-643b-4650-a036-6f1b9c3c8d03" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 224.043s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 798.781440] env[63273]: DEBUG nova.compute.manager [None req-b42cefbb-26e0-40af-9324-00e8a9e4b861 tempest-TenantUsagesTestJSON-1769228805 tempest-TenantUsagesTestJSON-1769228805-project-member] [instance: 7d000257-5fc1-43fa-9fc5-a07519a6feea] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 798.804708] env[63273]: DEBUG nova.network.neutron [-] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 798.818609] env[63273]: INFO nova.compute.manager [-] [instance: 02ed8dca-f6ae-47b0-a047-f89f858f5ff2] Took 0.08 seconds to deallocate network for instance. [ 798.840917] env[63273]: DEBUG nova.compute.manager [None req-b42cefbb-26e0-40af-9324-00e8a9e4b861 tempest-TenantUsagesTestJSON-1769228805 tempest-TenantUsagesTestJSON-1769228805-project-member] [instance: 7d000257-5fc1-43fa-9fc5-a07519a6feea] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 798.871480] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b42cefbb-26e0-40af-9324-00e8a9e4b861 tempest-TenantUsagesTestJSON-1769228805 tempest-TenantUsagesTestJSON-1769228805-project-member] Lock "7d000257-5fc1-43fa-9fc5-a07519a6feea" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 221.030s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 798.894236] env[63273]: DEBUG nova.compute.manager [None req-7c3e6778-2ab0-4c2b-9b3a-29c19c09b3bb tempest-ListImageFiltersTestJSON-45654626 tempest-ListImageFiltersTestJSON-45654626-project-member] [instance: 68fe9a49-e561-45da-afcb-e804e2109e76] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 798.955746] env[63273]: DEBUG nova.compute.manager [None req-7c3e6778-2ab0-4c2b-9b3a-29c19c09b3bb tempest-ListImageFiltersTestJSON-45654626 tempest-ListImageFiltersTestJSON-45654626-project-member] [instance: 68fe9a49-e561-45da-afcb-e804e2109e76] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 798.992908] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7c3e6778-2ab0-4c2b-9b3a-29c19c09b3bb tempest-ListImageFiltersTestJSON-45654626 tempest-ListImageFiltersTestJSON-45654626-project-member] Lock "68fe9a49-e561-45da-afcb-e804e2109e76" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 220.020s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 798.997056] env[63273]: DEBUG oslo_concurrency.lockutils [None req-75d9671d-8e48-4f02-8379-6099e1a21b35 tempest-ServerDiagnosticsNegativeTest-1399727866 tempest-ServerDiagnosticsNegativeTest-1399727866-project-member] Lock "02ed8dca-f6ae-47b0-a047-f89f858f5ff2" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.327s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.008735] env[63273]: DEBUG nova.compute.manager [None req-e89c7380-dba5-40be-bfcb-bc0b02bea0fa tempest-ServersWithSpecificFlavorTestJSON-1103463149 tempest-ServersWithSpecificFlavorTestJSON-1103463149-project-member] [instance: 8dd5ad91-d09a-41dd-a8d0-fd22e96fc279] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 799.047132] env[63273]: DEBUG nova.compute.manager [None req-e89c7380-dba5-40be-bfcb-bc0b02bea0fa tempest-ServersWithSpecificFlavorTestJSON-1103463149 tempest-ServersWithSpecificFlavorTestJSON-1103463149-project-member] [instance: 8dd5ad91-d09a-41dd-a8d0-fd22e96fc279] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 799.077909] env[63273]: DEBUG oslo_concurrency.lockutils [None req-e89c7380-dba5-40be-bfcb-bc0b02bea0fa tempest-ServersWithSpecificFlavorTestJSON-1103463149 tempest-ServersWithSpecificFlavorTestJSON-1103463149-project-member] Lock "8dd5ad91-d09a-41dd-a8d0-fd22e96fc279" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 219.672s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.092944] env[63273]: DEBUG nova.compute.manager [None req-2d4f2721-535d-4a81-a48b-76c005a769e7 tempest-ListImageFiltersTestJSON-45654626 tempest-ListImageFiltersTestJSON-45654626-project-member] [instance: 8728833b-c3ac-49b2-9a33-ecfae9597e9e] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 799.126939] env[63273]: DEBUG nova.compute.manager [None req-2d4f2721-535d-4a81-a48b-76c005a769e7 tempest-ListImageFiltersTestJSON-45654626 tempest-ListImageFiltersTestJSON-45654626-project-member] [instance: 8728833b-c3ac-49b2-9a33-ecfae9597e9e] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 799.161354] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2d4f2721-535d-4a81-a48b-76c005a769e7 tempest-ListImageFiltersTestJSON-45654626 tempest-ListImageFiltersTestJSON-45654626-project-member] Lock "8728833b-c3ac-49b2-9a33-ecfae9597e9e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 218.726s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.176270] env[63273]: DEBUG nova.compute.manager [None req-0ec976f8-3e01-4399-a146-318088183878 tempest-VolumesAssistedSnapshotsTest-905484168 tempest-VolumesAssistedSnapshotsTest-905484168-project-member] [instance: 6699f6e7-203d-4d8a-bc9e-8c8d5427db36] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 799.219688] env[63273]: DEBUG nova.compute.manager [None req-0ec976f8-3e01-4399-a146-318088183878 tempest-VolumesAssistedSnapshotsTest-905484168 tempest-VolumesAssistedSnapshotsTest-905484168-project-member] [instance: 6699f6e7-203d-4d8a-bc9e-8c8d5427db36] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 799.267919] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0ec976f8-3e01-4399-a146-318088183878 tempest-VolumesAssistedSnapshotsTest-905484168 tempest-VolumesAssistedSnapshotsTest-905484168-project-member] Lock "6699f6e7-203d-4d8a-bc9e-8c8d5427db36" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 216.233s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.284352] env[63273]: DEBUG nova.compute.manager [None req-27626105-34c4-40c9-8bcd-2c38276131a3 tempest-ServersTestBootFromVolume-777103340 tempest-ServersTestBootFromVolume-777103340-project-member] [instance: 362b7115-a270-407f-a53c-506c863c8fe1] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 799.328704] env[63273]: DEBUG nova.compute.manager [None req-27626105-34c4-40c9-8bcd-2c38276131a3 tempest-ServersTestBootFromVolume-777103340 tempest-ServersTestBootFromVolume-777103340-project-member] [instance: 362b7115-a270-407f-a53c-506c863c8fe1] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 799.415852] env[63273]: DEBUG oslo_concurrency.lockutils [None req-27626105-34c4-40c9-8bcd-2c38276131a3 tempest-ServersTestBootFromVolume-777103340 tempest-ServersTestBootFromVolume-777103340-project-member] Lock "362b7115-a270-407f-a53c-506c863c8fe1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 214.449s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.430272] env[63273]: DEBUG nova.compute.manager [None req-c402fea3-ce13-4ff0-864f-0f1bdb3edb6e tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] [instance: cc6f5d6c-1f3c-4ead-8ee4-9b59923ac57b] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 799.480299] env[63273]: DEBUG nova.compute.manager [None req-c402fea3-ce13-4ff0-864f-0f1bdb3edb6e tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] [instance: cc6f5d6c-1f3c-4ead-8ee4-9b59923ac57b] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 799.539559] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c402fea3-ce13-4ff0-864f-0f1bdb3edb6e tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] Lock "cc6f5d6c-1f3c-4ead-8ee4-9b59923ac57b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 211.975s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.559497] env[63273]: DEBUG nova.compute.manager [None req-26ede9af-6526-4b79-b9b1-5be31b9f9dac tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] [instance: 811bed85-6a4d-4340-9e4f-a2acd86d921c] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 799.608026] env[63273]: DEBUG nova.compute.manager [None req-26ede9af-6526-4b79-b9b1-5be31b9f9dac tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] [instance: 811bed85-6a4d-4340-9e4f-a2acd86d921c] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 799.656696] env[63273]: DEBUG oslo_concurrency.lockutils [None req-26ede9af-6526-4b79-b9b1-5be31b9f9dac tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] Lock "811bed85-6a4d-4340-9e4f-a2acd86d921c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 210.640s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.671306] env[63273]: DEBUG nova.compute.manager [None req-aa94caae-be63-44fb-aa6d-314d5d081b13 tempest-ServerShowV247Test-1154040461 tempest-ServerShowV247Test-1154040461-project-member] [instance: 707e02eb-1471-4ddd-be9e-a41f33d42482] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 799.706707] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Acquiring lock "7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 799.707037] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Lock "7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 799.722103] env[63273]: DEBUG nova.compute.manager [None req-aa94caae-be63-44fb-aa6d-314d5d081b13 tempest-ServerShowV247Test-1154040461 tempest-ServerShowV247Test-1154040461-project-member] [instance: 707e02eb-1471-4ddd-be9e-a41f33d42482] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 799.768150] env[63273]: DEBUG oslo_concurrency.lockutils [None req-aa94caae-be63-44fb-aa6d-314d5d081b13 tempest-ServerShowV247Test-1154040461 tempest-ServerShowV247Test-1154040461-project-member] Lock "707e02eb-1471-4ddd-be9e-a41f33d42482" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 209.887s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.786674] env[63273]: DEBUG nova.compute.manager [None req-ce1ab873-88d5-48ef-ba5e-2f411f0a5350 tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] [instance: 4846a37e-b175-4f23-b8c1-34a6b124513d] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 799.842491] env[63273]: DEBUG nova.compute.manager [None req-ce1ab873-88d5-48ef-ba5e-2f411f0a5350 tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] [instance: 4846a37e-b175-4f23-b8c1-34a6b124513d] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 799.900279] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ce1ab873-88d5-48ef-ba5e-2f411f0a5350 tempest-ListServerFiltersTestJSON-2084546296 tempest-ListServerFiltersTestJSON-2084546296-project-member] Lock "4846a37e-b175-4f23-b8c1-34a6b124513d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 209.457s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.912600] env[63273]: DEBUG nova.compute.manager [None req-6c42ddf1-295d-4de0-b309-d4e3c257cc87 tempest-ServerShowV247Test-1154040461 tempest-ServerShowV247Test-1154040461-project-member] [instance: 11838596-1730-4c23-bbf2-3159d3c6d72d] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 799.962980] env[63273]: DEBUG nova.compute.manager [None req-6c42ddf1-295d-4de0-b309-d4e3c257cc87 tempest-ServerShowV247Test-1154040461 tempest-ServerShowV247Test-1154040461-project-member] [instance: 11838596-1730-4c23-bbf2-3159d3c6d72d] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 799.997034] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6c42ddf1-295d-4de0-b309-d4e3c257cc87 tempest-ServerShowV247Test-1154040461 tempest-ServerShowV247Test-1154040461-project-member] Lock "11838596-1730-4c23-bbf2-3159d3c6d72d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 207.116s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.018035] env[63273]: DEBUG nova.compute.manager [None req-25fa772e-cd3c-434f-b968-53f657928c1b tempest-ServerExternalEventsTest-1759374695 tempest-ServerExternalEventsTest-1759374695-project-member] [instance: 717fd312-ac1d-42b6-9f93-13b9f235bd23] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 800.058660] env[63273]: DEBUG nova.compute.manager [None req-25fa772e-cd3c-434f-b968-53f657928c1b tempest-ServerExternalEventsTest-1759374695 tempest-ServerExternalEventsTest-1759374695-project-member] [instance: 717fd312-ac1d-42b6-9f93-13b9f235bd23] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 800.104828] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25fa772e-cd3c-434f-b968-53f657928c1b tempest-ServerExternalEventsTest-1759374695 tempest-ServerExternalEventsTest-1759374695-project-member] Lock "717fd312-ac1d-42b6-9f93-13b9f235bd23" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 207.221s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.117430] env[63273]: DEBUG nova.compute.manager [None req-530971e6-1c95-4e35-8faa-c89e61642e17 tempest-ServerMetadataNegativeTestJSON-1866644747 tempest-ServerMetadataNegativeTestJSON-1866644747-project-member] [instance: a12ff1bf-8cfa-4d03-828d-6b34727f61bf] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 800.159866] env[63273]: DEBUG nova.compute.manager [None req-530971e6-1c95-4e35-8faa-c89e61642e17 tempest-ServerMetadataNegativeTestJSON-1866644747 tempest-ServerMetadataNegativeTestJSON-1866644747-project-member] [instance: a12ff1bf-8cfa-4d03-828d-6b34727f61bf] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 800.188513] env[63273]: DEBUG oslo_concurrency.lockutils [None req-530971e6-1c95-4e35-8faa-c89e61642e17 tempest-ServerMetadataNegativeTestJSON-1866644747 tempest-ServerMetadataNegativeTestJSON-1866644747-project-member] Lock "a12ff1bf-8cfa-4d03-828d-6b34727f61bf" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 202.878s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.203615] env[63273]: DEBUG nova.compute.manager [None req-294aff5a-b2ea-40b8-885c-f8bde42d2b30 tempest-InstanceActionsNegativeTestJSON-4733633 tempest-InstanceActionsNegativeTestJSON-4733633-project-member] [instance: f889087d-5f22-4de6-8dff-819d2afb7240] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 800.236498] env[63273]: DEBUG nova.compute.manager [None req-294aff5a-b2ea-40b8-885c-f8bde42d2b30 tempest-InstanceActionsNegativeTestJSON-4733633 tempest-InstanceActionsNegativeTestJSON-4733633-project-member] [instance: f889087d-5f22-4de6-8dff-819d2afb7240] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 800.268939] env[63273]: DEBUG oslo_concurrency.lockutils [None req-294aff5a-b2ea-40b8-885c-f8bde42d2b30 tempest-InstanceActionsNegativeTestJSON-4733633 tempest-InstanceActionsNegativeTestJSON-4733633-project-member] Lock "f889087d-5f22-4de6-8dff-819d2afb7240" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 201.852s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.284489] env[63273]: DEBUG nova.compute.manager [None req-9745a7f0-5825-4992-90f1-1e5f105cf947 tempest-ServerActionsTestJSON-2016287253 tempest-ServerActionsTestJSON-2016287253-project-member] [instance: a4e56801-a8b8-448f-9b1a-d228ceeaaa46] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 800.317279] env[63273]: DEBUG nova.compute.manager [None req-9745a7f0-5825-4992-90f1-1e5f105cf947 tempest-ServerActionsTestJSON-2016287253 tempest-ServerActionsTestJSON-2016287253-project-member] [instance: a4e56801-a8b8-448f-9b1a-d228ceeaaa46] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 800.367293] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9745a7f0-5825-4992-90f1-1e5f105cf947 tempest-ServerActionsTestJSON-2016287253 tempest-ServerActionsTestJSON-2016287253-project-member] Lock "a4e56801-a8b8-448f-9b1a-d228ceeaaa46" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 199.950s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.417738] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 800.537921] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.537921] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.539961] env[63273]: INFO nova.compute.claims [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 800.890752] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 800.890972] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 800.907198] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 801.019027] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d537c80-323f-438c-9b40-0bc656e20670 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.034081] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98a36cc5-dc8e-4582-97f9-459cd48c5ee8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.067222] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41422af2-e5c8-458f-bcb1-494ea11c2601 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.075764] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7abf70d1-6a22-4b1a-a1ab-481cb5981bf3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.090526] env[63273]: DEBUG nova.compute.provider_tree [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 801.102872] env[63273]: DEBUG nova.scheduler.client.report [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 801.120656] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.585s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 801.121643] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 801.124790] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.220s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 801.125049] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 801.125224] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 801.126576] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6bb9b2e5-201a-4cd9-be4e-4280d4a39647 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.138443] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45cdb350-a6cd-4ba8-b5ff-cb85895e7ff0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.157650] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-347049ca-92e8-4e24-a976-ede7ba4d388a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.164360] env[63273]: DEBUG nova.compute.utils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 801.170018] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 801.170018] env[63273]: DEBUG nova.network.neutron [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 801.173620] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-613c1a58-e50e-48ab-b4ac-3b6343794642 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.212340] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180539MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 801.213924] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 801.213924] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 801.214667] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 801.304673] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 801.324993] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f1dddee3-33d6-4f55-bd91-9ce2fe6be72b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 801.326609] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 801.326609] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a5c37718-0067-4f5e-ba7e-533f13529739 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 801.326609] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance ab9cf87a-b74f-4b2f-842d-26e16b7fdb92 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 801.326609] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 94d19d7f-8eb7-4c93-aff4-3b1404338240 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 801.326920] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5501c192-867b-40c0-9bea-c3c44865d2f4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 801.326920] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77493b32-817c-4be8-a42d-8d48e707d41f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 801.326920] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 801.326920] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 801.327084] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 801.343306] env[63273]: DEBUG nova.virt.hardware [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 801.343727] env[63273]: DEBUG nova.virt.hardware [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 801.344056] env[63273]: DEBUG nova.virt.hardware [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 801.344937] env[63273]: DEBUG nova.virt.hardware [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 801.344937] env[63273]: DEBUG nova.virt.hardware [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 801.344937] env[63273]: DEBUG nova.virt.hardware [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 801.345109] env[63273]: DEBUG nova.virt.hardware [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 801.345239] env[63273]: DEBUG nova.virt.hardware [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 801.345436] env[63273]: DEBUG nova.virt.hardware [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 801.345613] env[63273]: DEBUG nova.virt.hardware [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 801.345764] env[63273]: DEBUG nova.virt.hardware [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 801.348927] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93020a66-77a6-4d52-a44f-fa6b7d344e20 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.352438] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1979e66a-172d-4130-90ea-9f0034b13071 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.360715] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b044cd25-98e2-4248-8c61-102fab74f125 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.367407] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 6b48c313-e3d5-4d3c-83c5-3c12d948abba has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.389617] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c936b733-7faa-4e73-99c5-2ac11d5421ff has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.401651] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance ff6cf180-c0fe-4b4f-9bac-08e9e3dc7c1d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.413801] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 02e9cc93-bdc3-4220-a22e-e29e959a5377 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.416817] env[63273]: DEBUG nova.policy [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '46f5b06c23d74c8b9bb08aa08b107367', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8189e2aa9960429596f8a65a91032bed', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 801.425811] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 451c95b6-a407-4911-bc40-83f2e2e7a75f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.439719] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e36789b5-f814-4105-b144-361fef9e0d0e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.457532] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.470868] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 13511c60-50cd-44ed-969e-c5fc29b0125e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.489572] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2eb7836e-f1ce-4eea-9cee-f9075cdf6d99 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.507890] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 3e73331f-ce47-4310-98dc-b1a9ac168d30 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.522737] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f5edcebb-61c8-47f5-a925-fd3f0b7a1389 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.534975] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 801.535257] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 801.535467] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '21', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '9', 'num_os_type_None': '10', 'num_proj_37d368bc456740739821f51e3a25f2fb': '1', 'io_workload': '10', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'num_proj_d806b4c5831b457fa075e387f55d8ad5': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_4b844f0a784d4f20a5adbf7354c75cc7': '1', 'num_proj_ad03061e544f4b17abb83d9596c461a0': '1', 'num_proj_06e0618687eb4144b9aacdc90a42879d': '1', 'num_proj_70158b9660d14db684be2828e94f8e42': '1', 'num_proj_f1ef99990442451e927a36cdbb924a86': '1', 'num_task_networking': '1', 'num_proj_8189e2aa9960429596f8a65a91032bed': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 801.942854] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad1b2080-a74f-4880-b076-f12d7835fcef {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.953910] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ef327f1-6163-4c49-be98-ceb53d162936 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.993438] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8a55aac-9de8-4c2d-b49a-0e682d4952f3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.003088] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d5d3bb3-7e5d-4ffd-8f2e-c005ce415a4e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.018360] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 802.041576] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 802.066928] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 802.067162] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.854s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.067672] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 802.067830] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11307}} [ 802.084916] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] There are 0 instances to clean {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11316}} [ 802.084995] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 802.085096] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances with incomplete migration {{(pid=63273) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11345}} [ 802.100249] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 802.502876] env[63273]: DEBUG nova.network.neutron [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Successfully created port: 9c58b21e-6c17-40b2-96b1-48123be3ab8e {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 803.111118] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 803.111118] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 803.891625] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 803.891625] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 804.752460] env[63273]: DEBUG nova.network.neutron [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Successfully updated port: 9c58b21e-6c17-40b2-96b1-48123be3ab8e {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 804.769563] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquiring lock "refresh_cache-164c1d04-3481-4aee-ba56-c80c8ed36e6a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 804.769962] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquired lock "refresh_cache-164c1d04-3481-4aee-ba56-c80c8ed36e6a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 804.771620] env[63273]: DEBUG nova.network.neutron [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 804.889464] env[63273]: DEBUG nova.network.neutron [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 804.894927] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 805.319610] env[63273]: DEBUG nova.network.neutron [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Updating instance_info_cache with network_info: [{"id": "9c58b21e-6c17-40b2-96b1-48123be3ab8e", "address": "fa:16:3e:96:db:c0", "network": {"id": "a9de121c-df01-46c2-b949-0624fe3878ca", "bridge": "br-int", "label": "tempest-ListServersNegativeTestJSON-1766145969-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "8189e2aa9960429596f8a65a91032bed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f8442aa5-73db-4599-8564-b98a6ea26b9b", "external-id": "nsx-vlan-transportzone-893", "segmentation_id": 893, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9c58b21e-6c", "ovs_interfaceid": "9c58b21e-6c17-40b2-96b1-48123be3ab8e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 805.354611] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Releasing lock "refresh_cache-164c1d04-3481-4aee-ba56-c80c8ed36e6a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 805.354928] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Instance network_info: |[{"id": "9c58b21e-6c17-40b2-96b1-48123be3ab8e", "address": "fa:16:3e:96:db:c0", "network": {"id": "a9de121c-df01-46c2-b949-0624fe3878ca", "bridge": "br-int", "label": "tempest-ListServersNegativeTestJSON-1766145969-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "8189e2aa9960429596f8a65a91032bed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f8442aa5-73db-4599-8564-b98a6ea26b9b", "external-id": "nsx-vlan-transportzone-893", "segmentation_id": 893, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9c58b21e-6c", "ovs_interfaceid": "9c58b21e-6c17-40b2-96b1-48123be3ab8e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 805.355431] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:96:db:c0', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f8442aa5-73db-4599-8564-b98a6ea26b9b', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '9c58b21e-6c17-40b2-96b1-48123be3ab8e', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 805.363878] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Creating folder: Project (8189e2aa9960429596f8a65a91032bed). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 805.364944] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-a815e9d0-f9ac-45ec-bd0e-76ea642496d5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 805.379018] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Created folder: Project (8189e2aa9960429596f8a65a91032bed) in parent group-v986930. [ 805.379246] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Creating folder: Instances. Parent ref: group-v986977. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 805.379512] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-3bf1bd40-6ecf-4180-a4d4-6fbaefd8dabc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 805.392279] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Created folder: Instances in parent group-v986977. [ 805.392687] env[63273]: DEBUG oslo.service.loopingcall [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 805.392942] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 805.393452] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-c9f7a3e8-7b0a-4bcc-8078-5eda05061a76 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 805.422630] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 805.422630] env[63273]: value = "task-5072043" [ 805.422630] env[63273]: _type = "Task" [ 805.422630] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 805.432635] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072043, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 805.521818] env[63273]: DEBUG nova.compute.manager [req-d9a3f341-3ff3-4810-b1b3-9007ab5d80bd req-ffc8324c-29a5-4b6b-8697-e71ab91724ff service nova] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Received event network-vif-plugged-9c58b21e-6c17-40b2-96b1-48123be3ab8e {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 805.522063] env[63273]: DEBUG oslo_concurrency.lockutils [req-d9a3f341-3ff3-4810-b1b3-9007ab5d80bd req-ffc8324c-29a5-4b6b-8697-e71ab91724ff service nova] Acquiring lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 805.522284] env[63273]: DEBUG oslo_concurrency.lockutils [req-d9a3f341-3ff3-4810-b1b3-9007ab5d80bd req-ffc8324c-29a5-4b6b-8697-e71ab91724ff service nova] Lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 805.522453] env[63273]: DEBUG oslo_concurrency.lockutils [req-d9a3f341-3ff3-4810-b1b3-9007ab5d80bd req-ffc8324c-29a5-4b6b-8697-e71ab91724ff service nova] Lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 805.522616] env[63273]: DEBUG nova.compute.manager [req-d9a3f341-3ff3-4810-b1b3-9007ab5d80bd req-ffc8324c-29a5-4b6b-8697-e71ab91724ff service nova] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] No waiting events found dispatching network-vif-plugged-9c58b21e-6c17-40b2-96b1-48123be3ab8e {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 805.522779] env[63273]: WARNING nova.compute.manager [req-d9a3f341-3ff3-4810-b1b3-9007ab5d80bd req-ffc8324c-29a5-4b6b-8697-e71ab91724ff service nova] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Received unexpected event network-vif-plugged-9c58b21e-6c17-40b2-96b1-48123be3ab8e for instance with vm_state building and task_state spawning. [ 805.889257] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 805.935458] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072043, 'name': CreateVM_Task, 'duration_secs': 0.383492} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 805.935458] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 805.936265] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 805.936524] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 805.936910] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 805.937232] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-0c9cc311-e7fb-493c-a836-a6c2e1a300c1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 805.943741] env[63273]: DEBUG oslo_vmware.api [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Waiting for the task: (returnval){ [ 805.943741] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]524203be-5917-0a26-8d43-7f2e855aacbe" [ 805.943741] env[63273]: _type = "Task" [ 805.943741] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 805.955021] env[63273]: DEBUG oslo_vmware.api [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]524203be-5917-0a26-8d43-7f2e855aacbe, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 806.456101] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 806.456377] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 806.456589] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 806.891281] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 806.891605] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 806.891605] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 806.930224] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 806.930224] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 806.930224] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 806.930224] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 806.930224] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 806.930434] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 806.930434] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 806.930434] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 806.930434] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 806.930434] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 806.930571] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 806.931056] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 808.576589] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquiring lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 808.710971] env[63273]: DEBUG nova.compute.manager [req-d907512c-c76f-43f4-9206-69062e4bbd1f req-119a1879-fae3-48bc-b22d-6e8f7e1d9be0 service nova] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Received event network-changed-9c58b21e-6c17-40b2-96b1-48123be3ab8e {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 808.712047] env[63273]: DEBUG nova.compute.manager [req-d907512c-c76f-43f4-9206-69062e4bbd1f req-119a1879-fae3-48bc-b22d-6e8f7e1d9be0 service nova] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Refreshing instance network info cache due to event network-changed-9c58b21e-6c17-40b2-96b1-48123be3ab8e. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 808.712047] env[63273]: DEBUG oslo_concurrency.lockutils [req-d907512c-c76f-43f4-9206-69062e4bbd1f req-119a1879-fae3-48bc-b22d-6e8f7e1d9be0 service nova] Acquiring lock "refresh_cache-164c1d04-3481-4aee-ba56-c80c8ed36e6a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 808.712047] env[63273]: DEBUG oslo_concurrency.lockutils [req-d907512c-c76f-43f4-9206-69062e4bbd1f req-119a1879-fae3-48bc-b22d-6e8f7e1d9be0 service nova] Acquired lock "refresh_cache-164c1d04-3481-4aee-ba56-c80c8ed36e6a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 808.712047] env[63273]: DEBUG nova.network.neutron [req-d907512c-c76f-43f4-9206-69062e4bbd1f req-119a1879-fae3-48bc-b22d-6e8f7e1d9be0 service nova] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Refreshing network info cache for port 9c58b21e-6c17-40b2-96b1-48123be3ab8e {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 809.085347] env[63273]: DEBUG nova.network.neutron [req-d907512c-c76f-43f4-9206-69062e4bbd1f req-119a1879-fae3-48bc-b22d-6e8f7e1d9be0 service nova] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Updated VIF entry in instance network info cache for port 9c58b21e-6c17-40b2-96b1-48123be3ab8e. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 809.085955] env[63273]: DEBUG nova.network.neutron [req-d907512c-c76f-43f4-9206-69062e4bbd1f req-119a1879-fae3-48bc-b22d-6e8f7e1d9be0 service nova] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Updating instance_info_cache with network_info: [{"id": "9c58b21e-6c17-40b2-96b1-48123be3ab8e", "address": "fa:16:3e:96:db:c0", "network": {"id": "a9de121c-df01-46c2-b949-0624fe3878ca", "bridge": "br-int", "label": "tempest-ListServersNegativeTestJSON-1766145969-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "8189e2aa9960429596f8a65a91032bed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f8442aa5-73db-4599-8564-b98a6ea26b9b", "external-id": "nsx-vlan-transportzone-893", "segmentation_id": 893, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9c58b21e-6c", "ovs_interfaceid": "9c58b21e-6c17-40b2-96b1-48123be3ab8e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 809.095662] env[63273]: DEBUG oslo_concurrency.lockutils [req-d907512c-c76f-43f4-9206-69062e4bbd1f req-119a1879-fae3-48bc-b22d-6e8f7e1d9be0 service nova] Releasing lock "refresh_cache-164c1d04-3481-4aee-ba56-c80c8ed36e6a" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 809.997433] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquiring lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 809.997692] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 812.263600] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7eda3871-e07e-4a99-a7d9-54904f21d085 tempest-AttachVolumeTestJSON-293439156 tempest-AttachVolumeTestJSON-293439156-project-member] Acquiring lock "5b3974c2-5be8-4983-a9d4-962c17889e57" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 812.263966] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7eda3871-e07e-4a99-a7d9-54904f21d085 tempest-AttachVolumeTestJSON-293439156 tempest-AttachVolumeTestJSON-293439156-project-member] Lock "5b3974c2-5be8-4983-a9d4-962c17889e57" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 813.727238] env[63273]: DEBUG oslo_concurrency.lockutils [None req-92e7eb84-c0c9-44d2-a93b-07f36b8d8135 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquiring lock "7ed225d1-db66-42c1-99f3-9f3ef4576cfb" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 813.727476] env[63273]: DEBUG oslo_concurrency.lockutils [None req-92e7eb84-c0c9-44d2-a93b-07f36b8d8135 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "7ed225d1-db66-42c1-99f3-9f3ef4576cfb" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 818.522634] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b1ca5531-4fc3-45c9-8d37-3c468e0e32d3 tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] Acquiring lock "7476adfd-033e-4b40-899d-1cfd67fd927e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 818.523015] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b1ca5531-4fc3-45c9-8d37-3c468e0e32d3 tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] Lock "7476adfd-033e-4b40-899d-1cfd67fd927e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 824.612082] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fa64e833-ace5-44b5-bb8d-5cd192a1e920 tempest-ServerActionsTestOtherA-2021597575 tempest-ServerActionsTestOtherA-2021597575-project-member] Acquiring lock "69983659-211d-42be-bb6b-f2ae08db0c9e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 824.612378] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fa64e833-ace5-44b5-bb8d-5cd192a1e920 tempest-ServerActionsTestOtherA-2021597575 tempest-ServerActionsTestOtherA-2021597575-project-member] Lock "69983659-211d-42be-bb6b-f2ae08db0c9e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.097072] env[63273]: DEBUG oslo_concurrency.lockutils [None req-e54ff226-ad2b-4659-ae44-20fc0e22e0e4 tempest-ServerRescueNegativeTestJSON-1207392049 tempest-ServerRescueNegativeTestJSON-1207392049-project-member] Acquiring lock "e6dff48e-986b-422c-82da-532a49b9ae36" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.097407] env[63273]: DEBUG oslo_concurrency.lockutils [None req-e54ff226-ad2b-4659-ae44-20fc0e22e0e4 tempest-ServerRescueNegativeTestJSON-1207392049 tempest-ServerRescueNegativeTestJSON-1207392049-project-member] Lock "e6dff48e-986b-422c-82da-532a49b9ae36" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.269362] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1f8d4d1a-fa4c-4247-88a2-bcb870e5009b tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Acquiring lock "fb03832c-8618-40f5-98f1-349b2ca3327e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.269362] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1f8d4d1a-fa4c-4247-88a2-bcb870e5009b tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Lock "fb03832c-8618-40f5-98f1-349b2ca3327e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.309122] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1f8d4d1a-fa4c-4247-88a2-bcb870e5009b tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Acquiring lock "a838375d-aa1b-4e7b-9d17-404d95365b7e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.309122] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1f8d4d1a-fa4c-4247-88a2-bcb870e5009b tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Lock "a838375d-aa1b-4e7b-9d17-404d95365b7e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 828.630980] env[63273]: DEBUG oslo_concurrency.lockutils [None req-36b44b09-5dfb-4bfa-9deb-4666c11732b7 tempest-ServerRescueNegativeTestJSON-1207392049 tempest-ServerRescueNegativeTestJSON-1207392049-project-member] Acquiring lock "c63e222c-c3f7-4543-8d99-e636f240b531" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 828.630980] env[63273]: DEBUG oslo_concurrency.lockutils [None req-36b44b09-5dfb-4bfa-9deb-4666c11732b7 tempest-ServerRescueNegativeTestJSON-1207392049 tempest-ServerRescueNegativeTestJSON-1207392049-project-member] Lock "c63e222c-c3f7-4543-8d99-e636f240b531" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 845.088515] env[63273]: WARNING oslo_vmware.rw_handles [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 845.088515] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 845.088515] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 845.088515] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 845.088515] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 845.088515] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 845.088515] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 845.088515] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 845.088515] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 845.088515] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 845.088515] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 845.088515] env[63273]: ERROR oslo_vmware.rw_handles [ 845.088515] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/fadf268e-8dc2-4b2b-9cbf-2d70f83a3cd2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 845.090090] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 845.090390] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Copying Virtual Disk [datastore1] vmware_temp/fadf268e-8dc2-4b2b-9cbf-2d70f83a3cd2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/fadf268e-8dc2-4b2b-9cbf-2d70f83a3cd2/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 845.090688] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-b6932548-0295-4496-be93-5d1781d5ce79 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.099527] env[63273]: DEBUG oslo_vmware.api [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Waiting for the task: (returnval){ [ 845.099527] env[63273]: value = "task-5072044" [ 845.099527] env[63273]: _type = "Task" [ 845.099527] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 845.108719] env[63273]: DEBUG oslo_vmware.api [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Task: {'id': task-5072044, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 845.611901] env[63273]: DEBUG oslo_vmware.exceptions [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 845.611901] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 845.612322] env[63273]: ERROR nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 845.612322] env[63273]: Faults: ['InvalidArgument'] [ 845.612322] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Traceback (most recent call last): [ 845.612322] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 845.612322] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] yield resources [ 845.612322] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 845.612322] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] self.driver.spawn(context, instance, image_meta, [ 845.612322] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 845.612322] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 845.612322] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 845.612322] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] self._fetch_image_if_missing(context, vi) [ 845.612322] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] image_cache(vi, tmp_image_ds_loc) [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] vm_util.copy_virtual_disk( [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] session._wait_for_task(vmdk_copy_task) [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] return self.wait_for_task(task_ref) [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] return evt.wait() [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] result = hub.switch() [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 845.612730] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] return self.greenlet.switch() [ 845.613162] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 845.613162] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] self.f(*self.args, **self.kw) [ 845.613162] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 845.613162] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] raise exceptions.translate_fault(task_info.error) [ 845.613162] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 845.613162] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Faults: ['InvalidArgument'] [ 845.613162] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] [ 845.613162] env[63273]: INFO nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Terminating instance [ 845.615425] env[63273]: DEBUG nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 845.615626] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 845.615917] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 845.616133] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 845.616872] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-609a30bf-5a72-4675-a389-63e3e0d767f0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.619574] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-5aa6e674-dcf5-4479-a6fb-425c5f6b799b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.626170] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 845.626400] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-2820f3c6-9cb4-4274-8c32-4ef765972f6c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.628659] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 845.628838] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 845.629801] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-52472cc4-31a1-4eee-a142-24e177aa5e41 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.634797] env[63273]: DEBUG oslo_vmware.api [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 845.634797] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]521816fe-0999-a4ce-fefd-45eed8398203" [ 845.634797] env[63273]: _type = "Task" [ 845.634797] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 845.649362] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 845.649601] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating directory with path [datastore1] vmware_temp/4192631f-9c1d-4903-9f46-88b059dd2cca/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 845.649832] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-605fb8cb-bc7c-461f-b0fa-3bc6258c1fe8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.670184] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created directory with path [datastore1] vmware_temp/4192631f-9c1d-4903-9f46-88b059dd2cca/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 845.670423] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Fetch image to [datastore1] vmware_temp/4192631f-9c1d-4903-9f46-88b059dd2cca/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 845.670641] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/4192631f-9c1d-4903-9f46-88b059dd2cca/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 845.671557] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a02e20ec-85ae-4eaa-ba7e-06c70afffd30 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.679630] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36b3fcd6-9658-40f5-bde1-120f7cb59d42 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.690262] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1340f6e-718c-49f7-af43-b5bfbf7df158 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.695904] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 845.696146] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 845.696325] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Deleting the datastore file [datastore1] f1dddee3-33d6-4f55-bd91-9ce2fe6be72b {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 845.697050] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-4d474d2f-57c7-473a-8d53-4a669ad3968e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.727519] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a206beef-0759-4d53-9f57-877bd0b704cb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.730769] env[63273]: DEBUG oslo_vmware.api [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Waiting for the task: (returnval){ [ 845.730769] env[63273]: value = "task-5072046" [ 845.730769] env[63273]: _type = "Task" [ 845.730769] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 845.736988] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-97c4cd86-2f25-4f34-8638-1a107371fd73 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.742185] env[63273]: DEBUG oslo_vmware.api [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Task: {'id': task-5072046, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 845.763437] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 845.821765] env[63273]: DEBUG oslo_vmware.rw_handles [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4192631f-9c1d-4903-9f46-88b059dd2cca/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 845.883408] env[63273]: DEBUG oslo_vmware.rw_handles [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 845.883700] env[63273]: DEBUG oslo_vmware.rw_handles [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4192631f-9c1d-4903-9f46-88b059dd2cca/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 846.240732] env[63273]: DEBUG oslo_vmware.api [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Task: {'id': task-5072046, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.071542} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 846.241033] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 846.241189] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 846.241362] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 846.241587] env[63273]: INFO nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Took 0.63 seconds to destroy the instance on the hypervisor. [ 846.243981] env[63273]: DEBUG nova.compute.claims [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 846.243981] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 846.244191] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 846.363700] env[63273]: DEBUG nova.scheduler.client.report [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Refreshing inventories for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 846.381835] env[63273]: DEBUG nova.scheduler.client.report [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Updating ProviderTree inventory for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 846.382239] env[63273]: DEBUG nova.compute.provider_tree [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Updating inventory in ProviderTree for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 846.396696] env[63273]: DEBUG nova.scheduler.client.report [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Refreshing aggregate associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, aggregates: None {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 846.418058] env[63273]: DEBUG nova.scheduler.client.report [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Refreshing trait associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NODE,COMPUTE_IMAGE_TYPE_ISO {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 846.776172] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3bc5b33e-5411-4e66-8131-37ffdcfa2a0d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 846.784051] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e3a5a2e-3165-46e5-b369-bc5cd773500b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 846.813329] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9c980b2-ebc1-4455-ac5c-08f32ad72ab6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 846.821854] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3556e35-0468-4142-b4b4-72d152e2a70a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 846.836438] env[63273]: DEBUG nova.compute.provider_tree [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 846.845631] env[63273]: DEBUG nova.scheduler.client.report [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 846.862273] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.618s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 846.862985] env[63273]: ERROR nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 846.862985] env[63273]: Faults: ['InvalidArgument'] [ 846.862985] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Traceback (most recent call last): [ 846.862985] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 846.862985] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] self.driver.spawn(context, instance, image_meta, [ 846.862985] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 846.862985] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 846.862985] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 846.862985] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] self._fetch_image_if_missing(context, vi) [ 846.862985] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 846.862985] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] image_cache(vi, tmp_image_ds_loc) [ 846.862985] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] vm_util.copy_virtual_disk( [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] session._wait_for_task(vmdk_copy_task) [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] return self.wait_for_task(task_ref) [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] return evt.wait() [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] result = hub.switch() [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] return self.greenlet.switch() [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 846.863385] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] self.f(*self.args, **self.kw) [ 846.863793] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 846.863793] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] raise exceptions.translate_fault(task_info.error) [ 846.863793] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 846.863793] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Faults: ['InvalidArgument'] [ 846.863793] env[63273]: ERROR nova.compute.manager [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] [ 846.863990] env[63273]: DEBUG nova.compute.utils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 846.865861] env[63273]: DEBUG nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Build of instance f1dddee3-33d6-4f55-bd91-9ce2fe6be72b was re-scheduled: A specified parameter was not correct: fileType [ 846.865861] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 846.866382] env[63273]: DEBUG nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 846.866629] env[63273]: DEBUG nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 846.866848] env[63273]: DEBUG nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 846.867092] env[63273]: DEBUG nova.network.neutron [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 847.341417] env[63273]: DEBUG nova.network.neutron [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 847.358230] env[63273]: INFO nova.compute.manager [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Took 0.49 seconds to deallocate network for instance. [ 847.480569] env[63273]: INFO nova.scheduler.client.report [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Deleted allocations for instance f1dddee3-33d6-4f55-bd91-9ce2fe6be72b [ 847.506428] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40ac4f31-e1bc-49bc-8279-85a0d9fd7e16 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 297.802s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.507706] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 283.232s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 847.507901] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] During sync_power_state the instance has a pending task (spawning). Skip. [ 847.508085] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.508706] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 98.742s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 847.508923] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Acquiring lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 847.509140] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 847.509315] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.511568] env[63273]: INFO nova.compute.manager [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Terminating instance [ 847.513344] env[63273]: DEBUG nova.compute.manager [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 847.513601] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 847.513862] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-15ebb226-22b1-426d-b50f-5ea77e52fae6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.519069] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 1979e66a-172d-4130-90ea-9f0034b13071] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 847.525449] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7399843-cdf2-4aaf-8f27-3b2156418ce7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.546148] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 1979e66a-172d-4130-90ea-9f0034b13071] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 847.559340] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f1dddee3-33d6-4f55-bd91-9ce2fe6be72b could not be found. [ 847.559562] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 847.559749] env[63273]: INFO nova.compute.manager [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Took 0.05 seconds to destroy the instance on the hypervisor. [ 847.560010] env[63273]: DEBUG oslo.service.loopingcall [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 847.562735] env[63273]: DEBUG nova.compute.manager [-] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 847.562865] env[63273]: DEBUG nova.network.neutron [-] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 847.575075] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "1979e66a-172d-4130-90ea-9f0034b13071" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 238.473s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.587024] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 6b48c313-e3d5-4d3c-83c5-3c12d948abba] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 847.613051] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 6b48c313-e3d5-4d3c-83c5-3c12d948abba] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 847.638448] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "6b48c313-e3d5-4d3c-83c5-3c12d948abba" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 238.502s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.651959] env[63273]: DEBUG nova.network.neutron [-] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 847.662556] env[63273]: DEBUG nova.compute.manager [None req-9be77a4a-85b3-4628-bb42-cb12147dbd02 tempest-VolumesAdminNegativeTest-184811883 tempest-VolumesAdminNegativeTest-184811883-project-member] [instance: c936b733-7faa-4e73-99c5-2ac11d5421ff] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 847.665458] env[63273]: INFO nova.compute.manager [-] [instance: f1dddee3-33d6-4f55-bd91-9ce2fe6be72b] Took 0.10 seconds to deallocate network for instance. [ 847.687272] env[63273]: DEBUG nova.compute.manager [None req-9be77a4a-85b3-4628-bb42-cb12147dbd02 tempest-VolumesAdminNegativeTest-184811883 tempest-VolumesAdminNegativeTest-184811883-project-member] [instance: c936b733-7faa-4e73-99c5-2ac11d5421ff] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 847.709614] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9be77a4a-85b3-4628-bb42-cb12147dbd02 tempest-VolumesAdminNegativeTest-184811883 tempest-VolumesAdminNegativeTest-184811883-project-member] Lock "c936b733-7faa-4e73-99c5-2ac11d5421ff" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 236.368s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.724052] env[63273]: DEBUG nova.compute.manager [None req-28203e26-125e-4639-bf38-b7aef42c13a5 tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] [instance: ff6cf180-c0fe-4b4f-9bac-08e9e3dc7c1d] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 847.752103] env[63273]: DEBUG nova.compute.manager [None req-28203e26-125e-4639-bf38-b7aef42c13a5 tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] [instance: ff6cf180-c0fe-4b4f-9bac-08e9e3dc7c1d] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 847.800543] env[63273]: DEBUG oslo_concurrency.lockutils [None req-28203e26-125e-4639-bf38-b7aef42c13a5 tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] Lock "ff6cf180-c0fe-4b4f-9bac-08e9e3dc7c1d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 232.533s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.802222] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ff1ca1c9-5777-45ea-853d-2c9088c63cb9 tempest-ServerDiagnosticsTest-973413048 tempest-ServerDiagnosticsTest-973413048-project-member] Lock "f1dddee3-33d6-4f55-bd91-9ce2fe6be72b" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.293s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.811294] env[63273]: DEBUG nova.compute.manager [None req-92deb72e-507e-4804-9869-474be9cc2156 tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] [instance: 02e9cc93-bdc3-4220-a22e-e29e959a5377] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 847.834251] env[63273]: DEBUG nova.compute.manager [None req-92deb72e-507e-4804-9869-474be9cc2156 tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] [instance: 02e9cc93-bdc3-4220-a22e-e29e959a5377] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 847.857491] env[63273]: DEBUG oslo_concurrency.lockutils [None req-92deb72e-507e-4804-9869-474be9cc2156 tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Lock "02e9cc93-bdc3-4220-a22e-e29e959a5377" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 220.629s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.871284] env[63273]: DEBUG nova.compute.manager [None req-92deb72e-507e-4804-9869-474be9cc2156 tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] [instance: 451c95b6-a407-4911-bc40-83f2e2e7a75f] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 847.899871] env[63273]: DEBUG nova.compute.manager [None req-92deb72e-507e-4804-9869-474be9cc2156 tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] [instance: 451c95b6-a407-4911-bc40-83f2e2e7a75f] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 847.922036] env[63273]: DEBUG oslo_concurrency.lockutils [None req-92deb72e-507e-4804-9869-474be9cc2156 tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Lock "451c95b6-a407-4911-bc40-83f2e2e7a75f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 220.658s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.935217] env[63273]: DEBUG nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 847.994046] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 847.994046] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 847.995643] env[63273]: INFO nova.compute.claims [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 848.360267] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a1e2258-2ad4-4c13-bf88-5d250d753942 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.370394] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76c0d9fd-c6d1-4122-9b2e-36bd03c5787f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.403517] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68ea2a0a-8415-41b3-9ea3-aa5b11994c0b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.411872] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e37c3f0b-a767-4ed8-b29d-3531b6454a66 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.427116] env[63273]: DEBUG nova.compute.provider_tree [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 848.438032] env[63273]: DEBUG nova.scheduler.client.report [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 848.453052] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.459s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 848.453627] env[63273]: DEBUG nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 848.491380] env[63273]: DEBUG nova.compute.utils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 848.492833] env[63273]: DEBUG nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 848.493032] env[63273]: DEBUG nova.network.neutron [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 848.504352] env[63273]: DEBUG nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 848.563695] env[63273]: DEBUG nova.policy [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6c0d90da405543039e3cefcaf9c70a3c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'bc297a27a74f4303b258579a0b803d81', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 848.584337] env[63273]: DEBUG nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 848.613381] env[63273]: DEBUG nova.virt.hardware [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 848.613693] env[63273]: DEBUG nova.virt.hardware [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 848.613881] env[63273]: DEBUG nova.virt.hardware [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 848.614841] env[63273]: DEBUG nova.virt.hardware [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 848.615060] env[63273]: DEBUG nova.virt.hardware [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 848.615228] env[63273]: DEBUG nova.virt.hardware [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 848.615449] env[63273]: DEBUG nova.virt.hardware [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 848.615612] env[63273]: DEBUG nova.virt.hardware [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 848.615783] env[63273]: DEBUG nova.virt.hardware [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 848.615953] env[63273]: DEBUG nova.virt.hardware [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 848.616142] env[63273]: DEBUG nova.virt.hardware [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 848.616999] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a26faeb8-dfb3-457d-af39-9323912f9a3d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.628542] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c496136-bc4b-4042-81e3-73a33fe0c019 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.040425] env[63273]: DEBUG nova.network.neutron [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Successfully created port: 5caf3d70-897c-435e-b9bd-7139a6f236e8 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 849.947077] env[63273]: DEBUG nova.compute.manager [req-68f0f0d2-cea3-4575-b7ce-128d296af6f2 req-796eef38-208a-4367-8fc6-2d6b924f1343 service nova] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Received event network-vif-plugged-5caf3d70-897c-435e-b9bd-7139a6f236e8 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 849.947317] env[63273]: DEBUG oslo_concurrency.lockutils [req-68f0f0d2-cea3-4575-b7ce-128d296af6f2 req-796eef38-208a-4367-8fc6-2d6b924f1343 service nova] Acquiring lock "e36789b5-f814-4105-b144-361fef9e0d0e-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 849.947501] env[63273]: DEBUG oslo_concurrency.lockutils [req-68f0f0d2-cea3-4575-b7ce-128d296af6f2 req-796eef38-208a-4367-8fc6-2d6b924f1343 service nova] Lock "e36789b5-f814-4105-b144-361fef9e0d0e-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 849.947670] env[63273]: DEBUG oslo_concurrency.lockutils [req-68f0f0d2-cea3-4575-b7ce-128d296af6f2 req-796eef38-208a-4367-8fc6-2d6b924f1343 service nova] Lock "e36789b5-f814-4105-b144-361fef9e0d0e-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 849.947862] env[63273]: DEBUG nova.compute.manager [req-68f0f0d2-cea3-4575-b7ce-128d296af6f2 req-796eef38-208a-4367-8fc6-2d6b924f1343 service nova] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] No waiting events found dispatching network-vif-plugged-5caf3d70-897c-435e-b9bd-7139a6f236e8 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 849.948008] env[63273]: WARNING nova.compute.manager [req-68f0f0d2-cea3-4575-b7ce-128d296af6f2 req-796eef38-208a-4367-8fc6-2d6b924f1343 service nova] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Received unexpected event network-vif-plugged-5caf3d70-897c-435e-b9bd-7139a6f236e8 for instance with vm_state building and task_state spawning. [ 850.001157] env[63273]: DEBUG nova.network.neutron [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Successfully updated port: 5caf3d70-897c-435e-b9bd-7139a6f236e8 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 850.013723] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquiring lock "refresh_cache-e36789b5-f814-4105-b144-361fef9e0d0e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 850.015072] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquired lock "refresh_cache-e36789b5-f814-4105-b144-361fef9e0d0e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 850.015268] env[63273]: DEBUG nova.network.neutron [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 850.097208] env[63273]: DEBUG nova.network.neutron [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 850.400495] env[63273]: DEBUG nova.network.neutron [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Updating instance_info_cache with network_info: [{"id": "5caf3d70-897c-435e-b9bd-7139a6f236e8", "address": "fa:16:3e:19:20:8f", "network": {"id": "98255b90-7b86-4a9e-b42c-e6e4c5d947c5", "bridge": "br-int", "label": "tempest-ServersTestManualDisk-104952524-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "bc297a27a74f4303b258579a0b803d81", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c8edfde4-5a99-4745-956d-04da82ab1b85", "external-id": "nsx-vlan-transportzone-519", "segmentation_id": 519, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5caf3d70-89", "ovs_interfaceid": "5caf3d70-897c-435e-b9bd-7139a6f236e8", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 850.416028] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Releasing lock "refresh_cache-e36789b5-f814-4105-b144-361fef9e0d0e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 850.416256] env[63273]: DEBUG nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Instance network_info: |[{"id": "5caf3d70-897c-435e-b9bd-7139a6f236e8", "address": "fa:16:3e:19:20:8f", "network": {"id": "98255b90-7b86-4a9e-b42c-e6e4c5d947c5", "bridge": "br-int", "label": "tempest-ServersTestManualDisk-104952524-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "bc297a27a74f4303b258579a0b803d81", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c8edfde4-5a99-4745-956d-04da82ab1b85", "external-id": "nsx-vlan-transportzone-519", "segmentation_id": 519, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5caf3d70-89", "ovs_interfaceid": "5caf3d70-897c-435e-b9bd-7139a6f236e8", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 850.416709] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:19:20:8f', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'c8edfde4-5a99-4745-956d-04da82ab1b85', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '5caf3d70-897c-435e-b9bd-7139a6f236e8', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 850.424109] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Creating folder: Project (bc297a27a74f4303b258579a0b803d81). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 850.424760] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-81fd56be-fa4d-4432-af45-2fd6649b1bd6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.497787] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Created folder: Project (bc297a27a74f4303b258579a0b803d81) in parent group-v986930. [ 850.498040] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Creating folder: Instances. Parent ref: group-v986980. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 850.499036] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-317d193c-e30d-49e0-b1c4-0acfdf7f50df {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.507681] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Created folder: Instances in parent group-v986980. [ 850.507927] env[63273]: DEBUG oslo.service.loopingcall [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 850.508164] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 850.508615] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-495818ba-5365-4ad2-944f-b612d53ae656 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.528404] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 850.528404] env[63273]: value = "task-5072049" [ 850.528404] env[63273]: _type = "Task" [ 850.528404] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 850.537821] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072049, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 851.038204] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072049, 'name': CreateVM_Task, 'duration_secs': 0.312548} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 851.038580] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 851.039074] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 851.039241] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 851.039556] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 851.039806] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c5a0bc69-85b3-46ab-ab6c-277bd12f25ac {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.045079] env[63273]: DEBUG oslo_vmware.api [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Waiting for the task: (returnval){ [ 851.045079] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]529aa292-e380-4b13-e10a-32db281375d2" [ 851.045079] env[63273]: _type = "Task" [ 851.045079] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 851.053254] env[63273]: DEBUG oslo_vmware.api [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]529aa292-e380-4b13-e10a-32db281375d2, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 851.557992] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 851.558367] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 851.558591] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 851.976765] env[63273]: DEBUG nova.compute.manager [req-59173652-79ca-4f61-b6b8-d173c9f53c49 req-8449c556-b10d-4fc6-ae4d-dddf0c20c19b service nova] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Received event network-changed-5caf3d70-897c-435e-b9bd-7139a6f236e8 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 851.977303] env[63273]: DEBUG nova.compute.manager [req-59173652-79ca-4f61-b6b8-d173c9f53c49 req-8449c556-b10d-4fc6-ae4d-dddf0c20c19b service nova] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Refreshing instance network info cache due to event network-changed-5caf3d70-897c-435e-b9bd-7139a6f236e8. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 851.977303] env[63273]: DEBUG oslo_concurrency.lockutils [req-59173652-79ca-4f61-b6b8-d173c9f53c49 req-8449c556-b10d-4fc6-ae4d-dddf0c20c19b service nova] Acquiring lock "refresh_cache-e36789b5-f814-4105-b144-361fef9e0d0e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 851.977428] env[63273]: DEBUG oslo_concurrency.lockutils [req-59173652-79ca-4f61-b6b8-d173c9f53c49 req-8449c556-b10d-4fc6-ae4d-dddf0c20c19b service nova] Acquired lock "refresh_cache-e36789b5-f814-4105-b144-361fef9e0d0e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 851.977587] env[63273]: DEBUG nova.network.neutron [req-59173652-79ca-4f61-b6b8-d173c9f53c49 req-8449c556-b10d-4fc6-ae4d-dddf0c20c19b service nova] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Refreshing network info cache for port 5caf3d70-897c-435e-b9bd-7139a6f236e8 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 852.492258] env[63273]: DEBUG nova.network.neutron [req-59173652-79ca-4f61-b6b8-d173c9f53c49 req-8449c556-b10d-4fc6-ae4d-dddf0c20c19b service nova] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Updated VIF entry in instance network info cache for port 5caf3d70-897c-435e-b9bd-7139a6f236e8. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 852.492615] env[63273]: DEBUG nova.network.neutron [req-59173652-79ca-4f61-b6b8-d173c9f53c49 req-8449c556-b10d-4fc6-ae4d-dddf0c20c19b service nova] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Updating instance_info_cache with network_info: [{"id": "5caf3d70-897c-435e-b9bd-7139a6f236e8", "address": "fa:16:3e:19:20:8f", "network": {"id": "98255b90-7b86-4a9e-b42c-e6e4c5d947c5", "bridge": "br-int", "label": "tempest-ServersTestManualDisk-104952524-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "bc297a27a74f4303b258579a0b803d81", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c8edfde4-5a99-4745-956d-04da82ab1b85", "external-id": "nsx-vlan-transportzone-519", "segmentation_id": 519, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5caf3d70-89", "ovs_interfaceid": "5caf3d70-897c-435e-b9bd-7139a6f236e8", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 852.503216] env[63273]: DEBUG oslo_concurrency.lockutils [req-59173652-79ca-4f61-b6b8-d173c9f53c49 req-8449c556-b10d-4fc6-ae4d-dddf0c20c19b service nova] Releasing lock "refresh_cache-e36789b5-f814-4105-b144-361fef9e0d0e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 860.891674] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 862.891661] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 862.892014] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 862.904159] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 862.904386] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 862.904569] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 862.904922] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 862.905840] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-917e587d-a1da-4faf-82f0-ce07df3adf7c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.915111] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-702ed320-14a0-48c6-af8e-7c17e17f1c00 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.929784] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d4321fc-b236-4a31-a7ad-2d13e57d78a2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.936920] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc6e3d0a-02f7-4fa5-9aca-43bafd337bf7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.968077] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180507MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 862.968240] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 862.968443] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 863.058758] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 863.058924] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a5c37718-0067-4f5e-ba7e-533f13529739 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 863.059093] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance ab9cf87a-b74f-4b2f-842d-26e16b7fdb92 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 863.059251] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 94d19d7f-8eb7-4c93-aff4-3b1404338240 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 863.059377] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5501c192-867b-40c0-9bea-c3c44865d2f4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 863.059496] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77493b32-817c-4be8-a42d-8d48e707d41f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 863.059613] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 863.059730] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 863.059844] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 863.059960] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e36789b5-f814-4105-b144-361fef9e0d0e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 863.074462] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.086632] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 13511c60-50cd-44ed-969e-c5fc29b0125e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.098105] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2eb7836e-f1ce-4eea-9cee-f9075cdf6d99 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.109632] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 3e73331f-ce47-4310-98dc-b1a9ac168d30 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.120655] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f5edcebb-61c8-47f5-a925-fd3f0b7a1389 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.132029] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.143724] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.154655] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5b3974c2-5be8-4983-a9d4-962c17889e57 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.165671] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7ed225d1-db66-42c1-99f3-9f3ef4576cfb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.178804] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7476adfd-033e-4b40-899d-1cfd67fd927e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.189944] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 69983659-211d-42be-bb6b-f2ae08db0c9e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.201645] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e6dff48e-986b-422c-82da-532a49b9ae36 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.212892] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fb03832c-8618-40f5-98f1-349b2ca3327e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.224034] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a838375d-aa1b-4e7b-9d17-404d95365b7e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.239790] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c63e222c-c3f7-4543-8d99-e636f240b531 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 863.240084] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 863.240255] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '28', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '9', 'num_os_type_None': '10', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'io_workload': '10', 'num_proj_d806b4c5831b457fa075e387f55d8ad5': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_4b844f0a784d4f20a5adbf7354c75cc7': '1', 'num_proj_ad03061e544f4b17abb83d9596c461a0': '1', 'num_proj_06e0618687eb4144b9aacdc90a42879d': '1', 'num_proj_70158b9660d14db684be2828e94f8e42': '1', 'num_proj_f1ef99990442451e927a36cdbb924a86': '1', 'num_proj_8189e2aa9960429596f8a65a91032bed': '1', 'num_task_spawning': '1', 'num_proj_bc297a27a74f4303b258579a0b803d81': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 863.561992] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1916c0fa-f3b2-4552-ac1f-2926363d6a66 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 863.571352] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbafe3c7-9350-4279-b05a-ed49810d187e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 863.602052] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cb3b362-ca22-4c5f-a449-b1df73f64f73 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 863.611028] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9bab9bca-629e-44d8-9196-06e90c772247 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 863.623328] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 863.632502] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 863.648279] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 863.648279] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.680s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.648336] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 865.891542] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 865.891985] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 865.892206] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 866.889009] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 866.914115] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 866.914430] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 866.914430] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 866.934592] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 866.934811] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 866.934984] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 866.935135] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 866.935263] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 866.935386] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 866.935508] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 866.935628] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 866.935752] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 866.935945] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 866.936017] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 866.936443] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 867.934130] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 875.103822] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquiring lock "e36789b5-f814-4105-b144-361fef9e0d0e" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 895.818620] env[63273]: WARNING oslo_vmware.rw_handles [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 895.818620] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 895.818620] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 895.818620] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 895.818620] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 895.818620] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 895.818620] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 895.818620] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 895.818620] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 895.818620] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 895.818620] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 895.818620] env[63273]: ERROR oslo_vmware.rw_handles [ 895.819433] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/4192631f-9c1d-4903-9f46-88b059dd2cca/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 895.821399] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 895.821702] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Copying Virtual Disk [datastore1] vmware_temp/4192631f-9c1d-4903-9f46-88b059dd2cca/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/4192631f-9c1d-4903-9f46-88b059dd2cca/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 895.822029] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-30f9601d-1bdd-4ce0-9fee-3b294da2bc17 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 895.831222] env[63273]: DEBUG oslo_vmware.api [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 895.831222] env[63273]: value = "task-5072050" [ 895.831222] env[63273]: _type = "Task" [ 895.831222] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 895.840402] env[63273]: DEBUG oslo_vmware.api [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072050, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 896.342398] env[63273]: DEBUG oslo_vmware.exceptions [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 896.342695] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 896.343335] env[63273]: ERROR nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 896.343335] env[63273]: Faults: ['InvalidArgument'] [ 896.343335] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Traceback (most recent call last): [ 896.343335] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 896.343335] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] yield resources [ 896.343335] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 896.343335] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] self.driver.spawn(context, instance, image_meta, [ 896.343335] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 896.343335] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] self._vmops.spawn(context, instance, image_meta, injected_files, [ 896.343335] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 896.343335] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] self._fetch_image_if_missing(context, vi) [ 896.343335] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] image_cache(vi, tmp_image_ds_loc) [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] vm_util.copy_virtual_disk( [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] session._wait_for_task(vmdk_copy_task) [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] return self.wait_for_task(task_ref) [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] return evt.wait() [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] result = hub.switch() [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 896.343728] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] return self.greenlet.switch() [ 896.344196] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 896.344196] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] self.f(*self.args, **self.kw) [ 896.344196] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 896.344196] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] raise exceptions.translate_fault(task_info.error) [ 896.344196] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 896.344196] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Faults: ['InvalidArgument'] [ 896.344196] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] [ 896.344196] env[63273]: INFO nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Terminating instance [ 896.345450] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 896.345564] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 896.346106] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-054273b6-81ad-4bbb-9cd8-4ea01ff57554 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.348182] env[63273]: DEBUG nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 896.348375] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 896.349159] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d04ee51d-f271-4e5b-ad3b-f1ecbf377d3f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.356899] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 896.357191] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-1fda0a40-338c-4fa3-8095-1325e1de83e3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.359607] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 896.359784] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 896.360782] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-58d8bd04-1be9-4fbe-9be8-8e2da13db9f6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.366208] env[63273]: DEBUG oslo_vmware.api [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Waiting for the task: (returnval){ [ 896.366208] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52bb0729-f9f2-7af5-60eb-51cf3ed4c629" [ 896.366208] env[63273]: _type = "Task" [ 896.366208] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 896.374227] env[63273]: DEBUG oslo_vmware.api [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52bb0729-f9f2-7af5-60eb-51cf3ed4c629, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 896.430433] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 896.430947] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 896.430947] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleting the datastore file [datastore1] ab9cf87a-b74f-4b2f-842d-26e16b7fdb92 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 896.431251] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-6501e5a7-08da-4449-9ea7-25b6d0635b7b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.438141] env[63273]: DEBUG oslo_vmware.api [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 896.438141] env[63273]: value = "task-5072052" [ 896.438141] env[63273]: _type = "Task" [ 896.438141] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 896.447461] env[63273]: DEBUG oslo_vmware.api [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072052, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 896.877050] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 896.877050] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Creating directory with path [datastore1] vmware_temp/a2c11629-27e6-45dc-9740-5d2a4fd32e02/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 896.877395] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-bae81c14-c134-4967-a881-03605e027f88 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.890115] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Created directory with path [datastore1] vmware_temp/a2c11629-27e6-45dc-9740-5d2a4fd32e02/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 896.890387] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Fetch image to [datastore1] vmware_temp/a2c11629-27e6-45dc-9740-5d2a4fd32e02/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 896.890579] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/a2c11629-27e6-45dc-9740-5d2a4fd32e02/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 896.891370] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ddf464f9-7e20-4b23-a5d2-3136f3a69592 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.898678] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74196820-32f8-4243-b6c4-1ae929f33e2e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.908069] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6ce4b00-9317-482e-896f-836657970f01 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.942019] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d171d54f-43e5-45f9-98c4-f2362c832c0a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.949991] env[63273]: DEBUG oslo_vmware.api [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072052, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.06673} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 896.951263] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 896.951551] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 896.951717] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 896.951940] env[63273]: INFO nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Took 0.60 seconds to destroy the instance on the hypervisor. [ 896.953929] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-143357a0-deb6-4591-a995-bb3ed394b964 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.956572] env[63273]: DEBUG nova.compute.claims [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 896.956823] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 896.957099] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 897.055457] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 897.116298] env[63273]: DEBUG oslo_vmware.rw_handles [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/a2c11629-27e6-45dc-9740-5d2a4fd32e02/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 897.178685] env[63273]: DEBUG oslo_vmware.rw_handles [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 897.178942] env[63273]: DEBUG oslo_vmware.rw_handles [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/a2c11629-27e6-45dc-9740-5d2a4fd32e02/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 897.394539] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4a41f3a-ccee-4723-833a-089db516f528 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.402737] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b582461-b65d-4480-8276-713f785e6858 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.432205] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6be66b0d-390a-4d2d-9960-4a8df878591b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.440239] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0fa24dcf-df19-4a81-8c19-668fa5a5c786 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.453961] env[63273]: DEBUG nova.compute.provider_tree [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 897.466044] env[63273]: DEBUG nova.scheduler.client.report [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 897.482066] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.525s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 897.482683] env[63273]: ERROR nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 897.482683] env[63273]: Faults: ['InvalidArgument'] [ 897.482683] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Traceback (most recent call last): [ 897.482683] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 897.482683] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] self.driver.spawn(context, instance, image_meta, [ 897.482683] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 897.482683] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] self._vmops.spawn(context, instance, image_meta, injected_files, [ 897.482683] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 897.482683] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] self._fetch_image_if_missing(context, vi) [ 897.482683] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 897.482683] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] image_cache(vi, tmp_image_ds_loc) [ 897.482683] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] vm_util.copy_virtual_disk( [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] session._wait_for_task(vmdk_copy_task) [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] return self.wait_for_task(task_ref) [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] return evt.wait() [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] result = hub.switch() [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] return self.greenlet.switch() [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 897.483089] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] self.f(*self.args, **self.kw) [ 897.483476] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 897.483476] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] raise exceptions.translate_fault(task_info.error) [ 897.483476] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 897.483476] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Faults: ['InvalidArgument'] [ 897.483476] env[63273]: ERROR nova.compute.manager [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] [ 897.483476] env[63273]: DEBUG nova.compute.utils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 897.484981] env[63273]: DEBUG nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Build of instance ab9cf87a-b74f-4b2f-842d-26e16b7fdb92 was re-scheduled: A specified parameter was not correct: fileType [ 897.484981] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 897.485373] env[63273]: DEBUG nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 897.485550] env[63273]: DEBUG nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 897.485716] env[63273]: DEBUG nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 897.485879] env[63273]: DEBUG nova.network.neutron [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 897.963685] env[63273]: DEBUG nova.network.neutron [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 897.980075] env[63273]: INFO nova.compute.manager [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Took 0.49 seconds to deallocate network for instance. [ 898.110421] env[63273]: INFO nova.scheduler.client.report [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleted allocations for instance ab9cf87a-b74f-4b2f-842d-26e16b7fdb92 [ 898.141262] env[63273]: DEBUG oslo_concurrency.lockutils [None req-83f2a493-2a57-40f9-8c83-4ee6e3d7f29e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 338.264s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 898.142500] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 333.866s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 898.142689] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] During sync_power_state the instance has a pending task (spawning). Skip. [ 898.142862] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 898.145767] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 137.335s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 898.146017] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 898.146231] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 898.146402] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 898.148691] env[63273]: INFO nova.compute.manager [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Terminating instance [ 898.151337] env[63273]: DEBUG nova.compute.manager [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 898.151537] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 898.151802] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-fbc73492-95bb-44cf-905a-541fef017949 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.162045] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ba2cd4b-7a05-4247-9f7f-450e456a45f4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.175450] env[63273]: DEBUG nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 898.198940] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ab9cf87a-b74f-4b2f-842d-26e16b7fdb92 could not be found. [ 898.199184] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 898.199371] env[63273]: INFO nova.compute.manager [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Took 0.05 seconds to destroy the instance on the hypervisor. [ 898.199712] env[63273]: DEBUG oslo.service.loopingcall [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 898.199988] env[63273]: DEBUG nova.compute.manager [-] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 898.200092] env[63273]: DEBUG nova.network.neutron [-] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 898.234610] env[63273]: DEBUG nova.network.neutron [-] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 898.251468] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 898.251468] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 898.251998] env[63273]: INFO nova.compute.claims [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 898.254463] env[63273]: INFO nova.compute.manager [-] [instance: ab9cf87a-b74f-4b2f-842d-26e16b7fdb92] Took 0.05 seconds to deallocate network for instance. [ 898.356978] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40552212-bfd9-45ac-bdcc-666a10f6ea8d tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "ab9cf87a-b74f-4b2f-842d-26e16b7fdb92" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.211s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 898.649036] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4afdeabf-fc70-42fc-bbb5-eb4864279a82 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.656916] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aadaab2a-39ea-4f00-8eaa-ac62078be183 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.687947] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-824190ff-33fd-4b67-8561-dad50d6fa510 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.696125] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-006c7d47-a812-4c9b-882c-1caeefb23e88 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.710419] env[63273]: DEBUG nova.compute.provider_tree [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 898.719847] env[63273]: DEBUG nova.scheduler.client.report [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 898.735802] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.485s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 898.736449] env[63273]: DEBUG nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 898.774283] env[63273]: DEBUG nova.compute.utils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 898.775716] env[63273]: DEBUG nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 898.775876] env[63273]: DEBUG nova.network.neutron [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 898.787942] env[63273]: DEBUG nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 898.857083] env[63273]: DEBUG nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 898.883828] env[63273]: DEBUG nova.virt.hardware [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 898.884019] env[63273]: DEBUG nova.virt.hardware [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 898.884132] env[63273]: DEBUG nova.virt.hardware [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 898.884451] env[63273]: DEBUG nova.virt.hardware [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 898.884623] env[63273]: DEBUG nova.virt.hardware [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 898.884774] env[63273]: DEBUG nova.virt.hardware [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 898.885040] env[63273]: DEBUG nova.virt.hardware [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 898.885195] env[63273]: DEBUG nova.virt.hardware [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 898.885369] env[63273]: DEBUG nova.virt.hardware [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 898.885534] env[63273]: DEBUG nova.virt.hardware [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 898.885706] env[63273]: DEBUG nova.virt.hardware [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 898.887292] env[63273]: DEBUG nova.policy [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fc7b37355b3a4348908330557abb2e40', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6a04825b924a482994b31b7066af5714', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 898.889791] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-885352d7-e5e2-47bf-a6f5-dc68dabf2aa3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.898838] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba8ce00e-f39c-4fee-83e3-39defcca262f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.119554] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 899.119854] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 899.210751] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 899.333522] env[63273]: DEBUG nova.network.neutron [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Successfully created port: 44913323-d361-4095-8324-0cd854d4e790 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 900.458286] env[63273]: DEBUG nova.compute.manager [req-c99d6d11-4616-4dd1-88ae-940e40e7b98e req-ad780807-d62a-4362-9dd0-edcc1d2d25b6 service nova] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Received event network-vif-plugged-44913323-d361-4095-8324-0cd854d4e790 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 900.458544] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99d6d11-4616-4dd1-88ae-940e40e7b98e req-ad780807-d62a-4362-9dd0-edcc1d2d25b6 service nova] Acquiring lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 900.458726] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99d6d11-4616-4dd1-88ae-940e40e7b98e req-ad780807-d62a-4362-9dd0-edcc1d2d25b6 service nova] Lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 900.458891] env[63273]: DEBUG oslo_concurrency.lockutils [req-c99d6d11-4616-4dd1-88ae-940e40e7b98e req-ad780807-d62a-4362-9dd0-edcc1d2d25b6 service nova] Lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 900.459624] env[63273]: DEBUG nova.compute.manager [req-c99d6d11-4616-4dd1-88ae-940e40e7b98e req-ad780807-d62a-4362-9dd0-edcc1d2d25b6 service nova] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] No waiting events found dispatching network-vif-plugged-44913323-d361-4095-8324-0cd854d4e790 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 900.459860] env[63273]: WARNING nova.compute.manager [req-c99d6d11-4616-4dd1-88ae-940e40e7b98e req-ad780807-d62a-4362-9dd0-edcc1d2d25b6 service nova] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Received unexpected event network-vif-plugged-44913323-d361-4095-8324-0cd854d4e790 for instance with vm_state building and task_state deleting. [ 900.488421] env[63273]: DEBUG nova.network.neutron [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Successfully updated port: 44913323-d361-4095-8324-0cd854d4e790 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 900.503641] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "refresh_cache-d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 900.504291] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquired lock "refresh_cache-d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 900.504291] env[63273]: DEBUG nova.network.neutron [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 900.563068] env[63273]: DEBUG nova.network.neutron [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 900.787708] env[63273]: DEBUG nova.network.neutron [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Updating instance_info_cache with network_info: [{"id": "44913323-d361-4095-8324-0cd854d4e790", "address": "fa:16:3e:ed:41:96", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.147", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap44913323-d3", "ovs_interfaceid": "44913323-d361-4095-8324-0cd854d4e790", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 900.803178] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Releasing lock "refresh_cache-d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 900.803495] env[63273]: DEBUG nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Instance network_info: |[{"id": "44913323-d361-4095-8324-0cd854d4e790", "address": "fa:16:3e:ed:41:96", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.147", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap44913323-d3", "ovs_interfaceid": "44913323-d361-4095-8324-0cd854d4e790", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 900.803978] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:ed:41:96', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f66f8375-4460-4acd-987b-acda72bfcf0d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '44913323-d361-4095-8324-0cd854d4e790', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 900.812521] env[63273]: DEBUG oslo.service.loopingcall [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 900.812521] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 900.812734] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-ac83d64d-d21a-45e5-9286-23568d7cd062 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.835256] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 900.835256] env[63273]: value = "task-5072053" [ 900.835256] env[63273]: _type = "Task" [ 900.835256] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 900.844097] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072053, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 901.348080] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072053, 'name': CreateVM_Task, 'duration_secs': 0.301914} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 901.348080] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 901.348292] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 901.348394] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 901.348674] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 901.349083] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f89e4206-5e1b-435d-abd7-6ec7c9cec3cc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.354514] env[63273]: DEBUG oslo_vmware.api [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Waiting for the task: (returnval){ [ 901.354514] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52c97ecc-e5db-b112-ca9a-d514904be41e" [ 901.354514] env[63273]: _type = "Task" [ 901.354514] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 901.363717] env[63273]: DEBUG oslo_vmware.api [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52c97ecc-e5db-b112-ca9a-d514904be41e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 901.867242] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 901.867517] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 901.867734] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 902.561131] env[63273]: DEBUG nova.compute.manager [req-c31944c7-4e17-4b6e-930f-a85f0e88ce05 req-bde7cae0-a62e-43f4-a443-3a48efef2741 service nova] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Received event network-changed-44913323-d361-4095-8324-0cd854d4e790 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 902.561334] env[63273]: DEBUG nova.compute.manager [req-c31944c7-4e17-4b6e-930f-a85f0e88ce05 req-bde7cae0-a62e-43f4-a443-3a48efef2741 service nova] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Refreshing instance network info cache due to event network-changed-44913323-d361-4095-8324-0cd854d4e790. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 902.561548] env[63273]: DEBUG oslo_concurrency.lockutils [req-c31944c7-4e17-4b6e-930f-a85f0e88ce05 req-bde7cae0-a62e-43f4-a443-3a48efef2741 service nova] Acquiring lock "refresh_cache-d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 902.561687] env[63273]: DEBUG oslo_concurrency.lockutils [req-c31944c7-4e17-4b6e-930f-a85f0e88ce05 req-bde7cae0-a62e-43f4-a443-3a48efef2741 service nova] Acquired lock "refresh_cache-d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 902.561848] env[63273]: DEBUG nova.network.neutron [req-c31944c7-4e17-4b6e-930f-a85f0e88ce05 req-bde7cae0-a62e-43f4-a443-3a48efef2741 service nova] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Refreshing network info cache for port 44913323-d361-4095-8324-0cd854d4e790 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 902.973998] env[63273]: DEBUG nova.network.neutron [req-c31944c7-4e17-4b6e-930f-a85f0e88ce05 req-bde7cae0-a62e-43f4-a443-3a48efef2741 service nova] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Updated VIF entry in instance network info cache for port 44913323-d361-4095-8324-0cd854d4e790. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 902.974424] env[63273]: DEBUG nova.network.neutron [req-c31944c7-4e17-4b6e-930f-a85f0e88ce05 req-bde7cae0-a62e-43f4-a443-3a48efef2741 service nova] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Updating instance_info_cache with network_info: [{"id": "44913323-d361-4095-8324-0cd854d4e790", "address": "fa:16:3e:ed:41:96", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.147", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap44913323-d3", "ovs_interfaceid": "44913323-d361-4095-8324-0cd854d4e790", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 902.986852] env[63273]: DEBUG oslo_concurrency.lockutils [req-c31944c7-4e17-4b6e-930f-a85f0e88ce05 req-bde7cae0-a62e-43f4-a443-3a48efef2741 service nova] Releasing lock "refresh_cache-d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 920.891652] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 923.891577] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 924.891613] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 924.907317] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 924.907572] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 924.907703] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 924.907859] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 924.908988] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c7f11cb-6081-455f-ae25-7aacb73a7806 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.918488] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ebb2a64-fc51-41cc-baf4-1a0d80606df4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.933048] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6dff6227-132c-4ca1-a53d-7a5c4536c018 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.940145] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31f482fe-59e6-40f7-b390-83795a17dc1e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.968818] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180521MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 924.968989] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 924.969207] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 925.087021] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 925.087021] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a5c37718-0067-4f5e-ba7e-533f13529739 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 925.087021] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 94d19d7f-8eb7-4c93-aff4-3b1404338240 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 925.087021] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5501c192-867b-40c0-9bea-c3c44865d2f4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 925.087287] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77493b32-817c-4be8-a42d-8d48e707d41f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 925.087287] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 925.087287] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 925.087416] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 925.087507] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e36789b5-f814-4105-b144-361fef9e0d0e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 925.087621] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 925.111367] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 13511c60-50cd-44ed-969e-c5fc29b0125e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.122650] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2eb7836e-f1ce-4eea-9cee-f9075cdf6d99 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.134756] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 3e73331f-ce47-4310-98dc-b1a9ac168d30 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.146084] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f5edcebb-61c8-47f5-a925-fd3f0b7a1389 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.157323] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.169216] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.180770] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5b3974c2-5be8-4983-a9d4-962c17889e57 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.192565] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7ed225d1-db66-42c1-99f3-9f3ef4576cfb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.206346] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7476adfd-033e-4b40-899d-1cfd67fd927e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.220032] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 69983659-211d-42be-bb6b-f2ae08db0c9e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.231140] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e6dff48e-986b-422c-82da-532a49b9ae36 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.246815] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fb03832c-8618-40f5-98f1-349b2ca3327e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.259266] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a838375d-aa1b-4e7b-9d17-404d95365b7e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.271753] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c63e222c-c3f7-4543-8d99-e636f240b531 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.284478] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 925.284478] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 925.284478] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '29', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '10', 'num_os_type_None': '10', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'io_workload': '10', 'num_proj_d806b4c5831b457fa075e387f55d8ad5': '1', 'num_proj_4b844f0a784d4f20a5adbf7354c75cc7': '1', 'num_proj_ad03061e544f4b17abb83d9596c461a0': '1', 'num_proj_06e0618687eb4144b9aacdc90a42879d': '1', 'num_proj_70158b9660d14db684be2828e94f8e42': '1', 'num_proj_f1ef99990442451e927a36cdbb924a86': '1', 'num_proj_8189e2aa9960429596f8a65a91032bed': '1', 'num_proj_bc297a27a74f4303b258579a0b803d81': '1', 'num_proj_6a04825b924a482994b31b7066af5714': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 925.617799] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6642b397-6266-459d-893c-189de9222385 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.626147] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ea5d284-03fd-4e6e-99d5-b44b1ebe0924 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.656152] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02c5e88d-7616-4ff0-9726-242aa6a740ac {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.666019] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55e3b46c-e8c3-4597-b344-c301f2b8d985 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.678965] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 925.687812] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 925.712328] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 925.712441] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.743s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 926.713740] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 926.891238] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 926.891992] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 927.891011] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 927.891278] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 928.892664] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 928.893060] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 928.893060] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 928.916906] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 928.917343] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 928.917541] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 928.917678] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 928.917804] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 928.917926] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 928.918058] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 928.918181] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 928.918298] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 928.918413] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 928.918532] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 929.912737] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 945.129556] env[63273]: WARNING oslo_vmware.rw_handles [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 945.129556] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 945.129556] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 945.129556] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 945.129556] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 945.129556] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 945.129556] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 945.129556] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 945.129556] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 945.129556] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 945.129556] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 945.129556] env[63273]: ERROR oslo_vmware.rw_handles [ 945.130102] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/a2c11629-27e6-45dc-9740-5d2a4fd32e02/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 945.131803] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 945.132064] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Copying Virtual Disk [datastore1] vmware_temp/a2c11629-27e6-45dc-9740-5d2a4fd32e02/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/a2c11629-27e6-45dc-9740-5d2a4fd32e02/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 945.132350] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-fe5affb1-3070-4aa4-aaef-2df74e40e82c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.142184] env[63273]: DEBUG oslo_vmware.api [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Waiting for the task: (returnval){ [ 945.142184] env[63273]: value = "task-5072054" [ 945.142184] env[63273]: _type = "Task" [ 945.142184] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 945.150392] env[63273]: DEBUG oslo_vmware.api [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Task: {'id': task-5072054, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 945.652542] env[63273]: DEBUG oslo_vmware.exceptions [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 945.652826] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 945.653381] env[63273]: ERROR nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 945.653381] env[63273]: Faults: ['InvalidArgument'] [ 945.653381] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Traceback (most recent call last): [ 945.653381] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 945.653381] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] yield resources [ 945.653381] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 945.653381] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] self.driver.spawn(context, instance, image_meta, [ 945.653381] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 945.653381] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 945.653381] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 945.653381] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] self._fetch_image_if_missing(context, vi) [ 945.653381] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] image_cache(vi, tmp_image_ds_loc) [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] vm_util.copy_virtual_disk( [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] session._wait_for_task(vmdk_copy_task) [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] return self.wait_for_task(task_ref) [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] return evt.wait() [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] result = hub.switch() [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 945.653646] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] return self.greenlet.switch() [ 945.653910] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 945.653910] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] self.f(*self.args, **self.kw) [ 945.653910] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 945.653910] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] raise exceptions.translate_fault(task_info.error) [ 945.653910] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 945.653910] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Faults: ['InvalidArgument'] [ 945.653910] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] [ 945.653910] env[63273]: INFO nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Terminating instance [ 945.655987] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 945.655987] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 945.655987] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0dff9666-8b71-4926-83ce-b2a2fac15ce4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.658245] env[63273]: DEBUG nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 945.658480] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 945.659181] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1e9be03-769d-4f4c-aa87-154145884510 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.667163] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 945.667386] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-5cea9e72-d1df-4e32-ae54-8a59be447f63 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.669802] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 945.669976] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 945.670959] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-540236ef-07ef-4aed-8e2c-2e59fb062cb4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.676072] env[63273]: DEBUG oslo_vmware.api [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Waiting for the task: (returnval){ [ 945.676072] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52a5bbe2-591c-7f6f-d5bb-d8e065c2bbc8" [ 945.676072] env[63273]: _type = "Task" [ 945.676072] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 945.684202] env[63273]: DEBUG oslo_vmware.api [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52a5bbe2-591c-7f6f-d5bb-d8e065c2bbc8, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 945.739256] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 945.739921] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 945.740151] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Deleting the datastore file [datastore1] 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 945.740559] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-f676a62f-584e-4e2d-b007-5b85efabf877 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.747483] env[63273]: DEBUG oslo_vmware.api [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Waiting for the task: (returnval){ [ 945.747483] env[63273]: value = "task-5072056" [ 945.747483] env[63273]: _type = "Task" [ 945.747483] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 945.756210] env[63273]: DEBUG oslo_vmware.api [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Task: {'id': task-5072056, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 946.187164] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 946.187511] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Creating directory with path [datastore1] vmware_temp/a743bf18-d208-48ad-94c5-f363167e93b5/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 946.187511] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-dd975353-591e-471c-8fd8-094f38d02e0a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.200233] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Created directory with path [datastore1] vmware_temp/a743bf18-d208-48ad-94c5-f363167e93b5/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 946.200453] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Fetch image to [datastore1] vmware_temp/a743bf18-d208-48ad-94c5-f363167e93b5/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 946.200629] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/a743bf18-d208-48ad-94c5-f363167e93b5/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 946.201464] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b9735ec-a9f1-4aae-b3f8-8dc8fba7d6b8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.210106] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db77801c-bf75-4fcb-9fc3-d46ee859b56c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.219775] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8000e98e-b2c8-4f7c-a132-f973b0ab469d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.255955] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a94740bc-5726-4924-940d-97e06a67782d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.264079] env[63273]: DEBUG oslo_vmware.api [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Task: {'id': task-5072056, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.076165} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 946.265840] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 946.266039] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 946.266217] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 946.266390] env[63273]: INFO nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Took 0.61 seconds to destroy the instance on the hypervisor. [ 946.268517] env[63273]: DEBUG nova.compute.claims [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 946.268685] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 946.268894] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 946.272261] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-facae613-f49d-404f-bb83-76628cade23c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.297150] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 946.352305] env[63273]: DEBUG oslo_vmware.rw_handles [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/a743bf18-d208-48ad-94c5-f363167e93b5/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 946.412233] env[63273]: DEBUG oslo_vmware.rw_handles [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 946.412424] env[63273]: DEBUG oslo_vmware.rw_handles [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/a743bf18-d208-48ad-94c5-f363167e93b5/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 946.696023] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d84f65b-dc30-44f6-8f43-e5b5d08d1059 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.704430] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7d6ecc0-a16d-4440-9b2d-e4567a443ee5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.734267] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b02af39-5003-4f21-b18a-7f9c8440f826 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.742273] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2bbc9f6f-236a-4393-964c-a5a23a16f985 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.756881] env[63273]: DEBUG nova.compute.provider_tree [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 946.770385] env[63273]: DEBUG nova.scheduler.client.report [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 946.792026] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.521s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 946.792026] env[63273]: ERROR nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 946.792026] env[63273]: Faults: ['InvalidArgument'] [ 946.792026] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Traceback (most recent call last): [ 946.792026] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 946.792026] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] self.driver.spawn(context, instance, image_meta, [ 946.792026] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 946.792026] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 946.792026] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 946.792026] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] self._fetch_image_if_missing(context, vi) [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] image_cache(vi, tmp_image_ds_loc) [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] vm_util.copy_virtual_disk( [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] session._wait_for_task(vmdk_copy_task) [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] return self.wait_for_task(task_ref) [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] return evt.wait() [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] result = hub.switch() [ 946.792401] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 946.792661] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] return self.greenlet.switch() [ 946.792661] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 946.792661] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] self.f(*self.args, **self.kw) [ 946.792661] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 946.792661] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] raise exceptions.translate_fault(task_info.error) [ 946.792661] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 946.792661] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Faults: ['InvalidArgument'] [ 946.792661] env[63273]: ERROR nova.compute.manager [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] [ 946.792661] env[63273]: DEBUG nova.compute.utils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 946.793805] env[63273]: DEBUG nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Build of instance 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b was re-scheduled: A specified parameter was not correct: fileType [ 946.793805] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 946.794435] env[63273]: DEBUG nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 946.794739] env[63273]: DEBUG nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 946.794990] env[63273]: DEBUG nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 946.795275] env[63273]: DEBUG nova.network.neutron [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 947.151558] env[63273]: DEBUG nova.network.neutron [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 947.162041] env[63273]: INFO nova.compute.manager [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Took 0.37 seconds to deallocate network for instance. [ 947.261566] env[63273]: INFO nova.scheduler.client.report [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Deleted allocations for instance 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b [ 947.285637] env[63273]: DEBUG oslo_concurrency.lockutils [None req-31d4d133-b18a-41ae-a32c-59988fad6d94 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 394.886s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 947.286917] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 383.011s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 947.287136] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] During sync_power_state the instance has a pending task (spawning). Skip. [ 947.287370] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 947.287934] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 194.253s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 947.289925] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 947.291622] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.002s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 947.291834] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.002s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 947.294055] env[63273]: INFO nova.compute.manager [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Terminating instance [ 947.296167] env[63273]: DEBUG nova.compute.manager [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 947.296388] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 947.297186] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1b869fe5-9032-47c1-8d89-caebe1985440 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.311045] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee5a39f0-ba39-46d6-b78f-bab49a0ee497 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.322085] env[63273]: DEBUG nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 947.345449] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b could not be found. [ 947.345703] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 947.345840] env[63273]: INFO nova.compute.manager [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Took 0.05 seconds to destroy the instance on the hypervisor. [ 947.346028] env[63273]: DEBUG oslo.service.loopingcall [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 947.346911] env[63273]: DEBUG nova.compute.manager [-] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 947.346911] env[63273]: DEBUG nova.network.neutron [-] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 947.386236] env[63273]: DEBUG nova.network.neutron [-] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 947.396473] env[63273]: INFO nova.compute.manager [-] [instance: 89da94ce-40ad-40e0-a4b9-15fd15d0ab9b] Took 0.05 seconds to deallocate network for instance. [ 947.397797] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 947.398027] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 947.399776] env[63273]: INFO nova.compute.claims [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 947.515774] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e608f2f-87b4-4615-81fc-3d3ba49afbf7 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "89da94ce-40ad-40e0-a4b9-15fd15d0ab9b" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.227s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 947.790056] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7837ae2-9d3f-4eee-8223-9bcbcb47222b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.799783] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-badeba9e-7670-409c-be0a-c9fccd559420 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.830820] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18497966-853c-42f8-aa74-538428f07e9d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.838951] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a73e05c-1037-4341-a27a-88658a4090f6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.854178] env[63273]: DEBUG nova.compute.provider_tree [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 947.863841] env[63273]: DEBUG nova.scheduler.client.report [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 947.877682] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.480s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 947.878232] env[63273]: DEBUG nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 947.913203] env[63273]: DEBUG nova.compute.utils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 947.914709] env[63273]: DEBUG nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 947.914883] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 947.930915] env[63273]: DEBUG nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 947.989558] env[63273]: DEBUG nova.policy [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'bcadcf265aab4f178c48d945073b3934', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '702825b6bec4497f89ef4091bd7af0da', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 947.997553] env[63273]: DEBUG nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 948.026409] env[63273]: DEBUG nova.virt.hardware [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 948.026654] env[63273]: DEBUG nova.virt.hardware [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 948.026809] env[63273]: DEBUG nova.virt.hardware [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 948.026989] env[63273]: DEBUG nova.virt.hardware [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 948.027153] env[63273]: DEBUG nova.virt.hardware [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 948.027297] env[63273]: DEBUG nova.virt.hardware [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 948.027566] env[63273]: DEBUG nova.virt.hardware [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 948.027753] env[63273]: DEBUG nova.virt.hardware [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 948.027951] env[63273]: DEBUG nova.virt.hardware [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 948.028153] env[63273]: DEBUG nova.virt.hardware [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 948.028826] env[63273]: DEBUG nova.virt.hardware [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 948.029718] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c99fbb2-d598-43e1-8680-50cff54aea8a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 948.039575] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d2db202-7063-48d5-a251-59dbc5c24531 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 948.364191] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Successfully created port: 2364fdc6-13f8-4243-a4d5-93298a9b5d99 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 948.840348] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Successfully created port: 9ea385da-a15d-4101-bcc3-0fe11cc3d6c3 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 948.866709] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "b2b1ee46-307f-40fa-8346-394a39a0a99b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 948.866932] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "b2b1ee46-307f-40fa-8346-394a39a0a99b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 949.526322] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Successfully created port: 8cd0733e-3bbf-4c4a-bc63-b486fd145afa {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 950.262186] env[63273]: DEBUG nova.compute.manager [req-d634e7b9-e511-4540-8813-71c333ff6055 req-6372854b-0ab8-49ce-9897-29237d57c0a9 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Received event network-vif-plugged-2364fdc6-13f8-4243-a4d5-93298a9b5d99 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 950.262445] env[63273]: DEBUG oslo_concurrency.lockutils [req-d634e7b9-e511-4540-8813-71c333ff6055 req-6372854b-0ab8-49ce-9897-29237d57c0a9 service nova] Acquiring lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 950.262675] env[63273]: DEBUG oslo_concurrency.lockutils [req-d634e7b9-e511-4540-8813-71c333ff6055 req-6372854b-0ab8-49ce-9897-29237d57c0a9 service nova] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 950.262817] env[63273]: DEBUG oslo_concurrency.lockutils [req-d634e7b9-e511-4540-8813-71c333ff6055 req-6372854b-0ab8-49ce-9897-29237d57c0a9 service nova] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 950.262980] env[63273]: DEBUG nova.compute.manager [req-d634e7b9-e511-4540-8813-71c333ff6055 req-6372854b-0ab8-49ce-9897-29237d57c0a9 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] No waiting events found dispatching network-vif-plugged-2364fdc6-13f8-4243-a4d5-93298a9b5d99 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 950.263162] env[63273]: WARNING nova.compute.manager [req-d634e7b9-e511-4540-8813-71c333ff6055 req-6372854b-0ab8-49ce-9897-29237d57c0a9 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Received unexpected event network-vif-plugged-2364fdc6-13f8-4243-a4d5-93298a9b5d99 for instance with vm_state building and task_state spawning. [ 950.369650] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Successfully updated port: 2364fdc6-13f8-4243-a4d5-93298a9b5d99 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 951.304395] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Successfully updated port: 9ea385da-a15d-4101-bcc3-0fe11cc3d6c3 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 952.253533] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Successfully updated port: 8cd0733e-3bbf-4c4a-bc63-b486fd145afa {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 952.263587] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 952.263867] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquired lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 952.264080] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 952.406087] env[63273]: DEBUG nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Received event network-changed-2364fdc6-13f8-4243-a4d5-93298a9b5d99 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 952.406400] env[63273]: DEBUG nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Refreshing instance network info cache due to event network-changed-2364fdc6-13f8-4243-a4d5-93298a9b5d99. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 952.407673] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Acquiring lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 952.562962] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 953.328620] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Updating instance_info_cache with network_info: [{"id": "2364fdc6-13f8-4243-a4d5-93298a9b5d99", "address": "fa:16:3e:8f:eb:90", "network": {"id": "190ec16f-ddf0-40c9-8de4-e7fb6daa6ee5", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-608361111", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.137", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f3c995e9-7f2f-420c-880a-d60da6e708ad", "external-id": "nsx-vlan-transportzone-166", "segmentation_id": 166, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap2364fdc6-13", "ovs_interfaceid": "2364fdc6-13f8-4243-a4d5-93298a9b5d99", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "9ea385da-a15d-4101-bcc3-0fe11cc3d6c3", "address": "fa:16:3e:88:23:d4", "network": {"id": "b2e91981-edca-4bc0-b508-9bc10e5e9e36", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-793876214", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.136", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dd7d0d95-6848-4e69-ac21-75f8db82a3b5", "external-id": "nsx-vlan-transportzone-272", "segmentation_id": 272, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9ea385da-a1", "ovs_interfaceid": "9ea385da-a15d-4101-bcc3-0fe11cc3d6c3", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "8cd0733e-3bbf-4c4a-bc63-b486fd145afa", "address": "fa:16:3e:88:f6:29", "network": {"id": "190ec16f-ddf0-40c9-8de4-e7fb6daa6ee5", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-608361111", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.248", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f3c995e9-7f2f-420c-880a-d60da6e708ad", "external-id": "nsx-vlan-transportzone-166", "segmentation_id": 166, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8cd0733e-3b", "ovs_interfaceid": "8cd0733e-3bbf-4c4a-bc63-b486fd145afa", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 953.348410] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Releasing lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 953.348776] env[63273]: DEBUG nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Instance network_info: |[{"id": "2364fdc6-13f8-4243-a4d5-93298a9b5d99", "address": "fa:16:3e:8f:eb:90", "network": {"id": "190ec16f-ddf0-40c9-8de4-e7fb6daa6ee5", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-608361111", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.137", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f3c995e9-7f2f-420c-880a-d60da6e708ad", "external-id": "nsx-vlan-transportzone-166", "segmentation_id": 166, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap2364fdc6-13", "ovs_interfaceid": "2364fdc6-13f8-4243-a4d5-93298a9b5d99", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "9ea385da-a15d-4101-bcc3-0fe11cc3d6c3", "address": "fa:16:3e:88:23:d4", "network": {"id": "b2e91981-edca-4bc0-b508-9bc10e5e9e36", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-793876214", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.136", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dd7d0d95-6848-4e69-ac21-75f8db82a3b5", "external-id": "nsx-vlan-transportzone-272", "segmentation_id": 272, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9ea385da-a1", "ovs_interfaceid": "9ea385da-a15d-4101-bcc3-0fe11cc3d6c3", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "8cd0733e-3bbf-4c4a-bc63-b486fd145afa", "address": "fa:16:3e:88:f6:29", "network": {"id": "190ec16f-ddf0-40c9-8de4-e7fb6daa6ee5", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-608361111", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.248", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f3c995e9-7f2f-420c-880a-d60da6e708ad", "external-id": "nsx-vlan-transportzone-166", "segmentation_id": 166, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8cd0733e-3b", "ovs_interfaceid": "8cd0733e-3bbf-4c4a-bc63-b486fd145afa", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 953.349103] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Acquired lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 953.349285] env[63273]: DEBUG nova.network.neutron [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Refreshing network info cache for port 2364fdc6-13f8-4243-a4d5-93298a9b5d99 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 953.350467] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:8f:eb:90', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f3c995e9-7f2f-420c-880a-d60da6e708ad', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '2364fdc6-13f8-4243-a4d5-93298a9b5d99', 'vif_model': 'vmxnet3'}, {'network_name': 'br-int', 'mac_address': 'fa:16:3e:88:23:d4', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'dd7d0d95-6848-4e69-ac21-75f8db82a3b5', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '9ea385da-a15d-4101-bcc3-0fe11cc3d6c3', 'vif_model': 'vmxnet3'}, {'network_name': 'br-int', 'mac_address': 'fa:16:3e:88:f6:29', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f3c995e9-7f2f-420c-880a-d60da6e708ad', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '8cd0733e-3bbf-4c4a-bc63-b486fd145afa', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 953.362426] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Creating folder: Project (702825b6bec4497f89ef4091bd7af0da). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 953.365659] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-5d82885e-25d7-4fd8-91b7-b40e320fe30f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.377798] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Created folder: Project (702825b6bec4497f89ef4091bd7af0da) in parent group-v986930. [ 953.378053] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Creating folder: Instances. Parent ref: group-v986984. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 953.378298] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-ae9f3564-ec18-427c-8a9d-4fe5205f2da8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.387805] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Created folder: Instances in parent group-v986984. [ 953.388056] env[63273]: DEBUG oslo.service.loopingcall [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 953.388246] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 953.388457] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-a6c85f95-f590-48ff-bcca-5421c86a7359 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.414441] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 953.414441] env[63273]: value = "task-5072059" [ 953.414441] env[63273]: _type = "Task" [ 953.414441] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 953.424419] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072059, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 953.765332] env[63273]: DEBUG nova.network.neutron [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Updated VIF entry in instance network info cache for port 2364fdc6-13f8-4243-a4d5-93298a9b5d99. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 953.765839] env[63273]: DEBUG nova.network.neutron [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Updating instance_info_cache with network_info: [{"id": "2364fdc6-13f8-4243-a4d5-93298a9b5d99", "address": "fa:16:3e:8f:eb:90", "network": {"id": "190ec16f-ddf0-40c9-8de4-e7fb6daa6ee5", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-608361111", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.137", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f3c995e9-7f2f-420c-880a-d60da6e708ad", "external-id": "nsx-vlan-transportzone-166", "segmentation_id": 166, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap2364fdc6-13", "ovs_interfaceid": "2364fdc6-13f8-4243-a4d5-93298a9b5d99", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "9ea385da-a15d-4101-bcc3-0fe11cc3d6c3", "address": "fa:16:3e:88:23:d4", "network": {"id": "b2e91981-edca-4bc0-b508-9bc10e5e9e36", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-793876214", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.136", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dd7d0d95-6848-4e69-ac21-75f8db82a3b5", "external-id": "nsx-vlan-transportzone-272", "segmentation_id": 272, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9ea385da-a1", "ovs_interfaceid": "9ea385da-a15d-4101-bcc3-0fe11cc3d6c3", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "8cd0733e-3bbf-4c4a-bc63-b486fd145afa", "address": "fa:16:3e:88:f6:29", "network": {"id": "190ec16f-ddf0-40c9-8de4-e7fb6daa6ee5", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-608361111", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.248", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f3c995e9-7f2f-420c-880a-d60da6e708ad", "external-id": "nsx-vlan-transportzone-166", "segmentation_id": 166, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8cd0733e-3b", "ovs_interfaceid": "8cd0733e-3bbf-4c4a-bc63-b486fd145afa", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 953.778619] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Releasing lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 953.778868] env[63273]: DEBUG nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Received event network-vif-plugged-9ea385da-a15d-4101-bcc3-0fe11cc3d6c3 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 953.779075] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Acquiring lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 953.779285] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 953.779449] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 953.779619] env[63273]: DEBUG nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] No waiting events found dispatching network-vif-plugged-9ea385da-a15d-4101-bcc3-0fe11cc3d6c3 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 953.779787] env[63273]: WARNING nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Received unexpected event network-vif-plugged-9ea385da-a15d-4101-bcc3-0fe11cc3d6c3 for instance with vm_state building and task_state spawning. [ 953.779947] env[63273]: DEBUG nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Received event network-changed-9ea385da-a15d-4101-bcc3-0fe11cc3d6c3 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 953.780118] env[63273]: DEBUG nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Refreshing instance network info cache due to event network-changed-9ea385da-a15d-4101-bcc3-0fe11cc3d6c3. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 953.780302] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Acquiring lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 953.780448] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Acquired lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 953.780606] env[63273]: DEBUG nova.network.neutron [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Refreshing network info cache for port 9ea385da-a15d-4101-bcc3-0fe11cc3d6c3 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 953.924795] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072059, 'name': CreateVM_Task, 'duration_secs': 0.416665} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 953.927014] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 953.927871] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 953.928052] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 953.928364] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 953.928908] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-71bdad2c-f303-4134-8ce4-9719b50fb3d8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.933610] env[63273]: DEBUG oslo_vmware.api [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Waiting for the task: (returnval){ [ 953.933610] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52059b70-b601-47e3-6897-234101359524" [ 953.933610] env[63273]: _type = "Task" [ 953.933610] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 953.941610] env[63273]: DEBUG oslo_vmware.api [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52059b70-b601-47e3-6897-234101359524, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 954.127629] env[63273]: DEBUG nova.network.neutron [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Updated VIF entry in instance network info cache for port 9ea385da-a15d-4101-bcc3-0fe11cc3d6c3. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 954.128127] env[63273]: DEBUG nova.network.neutron [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Updating instance_info_cache with network_info: [{"id": "2364fdc6-13f8-4243-a4d5-93298a9b5d99", "address": "fa:16:3e:8f:eb:90", "network": {"id": "190ec16f-ddf0-40c9-8de4-e7fb6daa6ee5", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-608361111", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.137", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f3c995e9-7f2f-420c-880a-d60da6e708ad", "external-id": "nsx-vlan-transportzone-166", "segmentation_id": 166, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap2364fdc6-13", "ovs_interfaceid": "2364fdc6-13f8-4243-a4d5-93298a9b5d99", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "9ea385da-a15d-4101-bcc3-0fe11cc3d6c3", "address": "fa:16:3e:88:23:d4", "network": {"id": "b2e91981-edca-4bc0-b508-9bc10e5e9e36", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-793876214", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.136", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dd7d0d95-6848-4e69-ac21-75f8db82a3b5", "external-id": "nsx-vlan-transportzone-272", "segmentation_id": 272, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9ea385da-a1", "ovs_interfaceid": "9ea385da-a15d-4101-bcc3-0fe11cc3d6c3", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "8cd0733e-3bbf-4c4a-bc63-b486fd145afa", "address": "fa:16:3e:88:f6:29", "network": {"id": "190ec16f-ddf0-40c9-8de4-e7fb6daa6ee5", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-608361111", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.248", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f3c995e9-7f2f-420c-880a-d60da6e708ad", "external-id": "nsx-vlan-transportzone-166", "segmentation_id": 166, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8cd0733e-3b", "ovs_interfaceid": "8cd0733e-3bbf-4c4a-bc63-b486fd145afa", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 954.138272] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Releasing lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 954.138528] env[63273]: DEBUG nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Received event network-vif-plugged-8cd0733e-3bbf-4c4a-bc63-b486fd145afa {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 954.138733] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Acquiring lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 954.138933] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 954.139105] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 954.139272] env[63273]: DEBUG nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] No waiting events found dispatching network-vif-plugged-8cd0733e-3bbf-4c4a-bc63-b486fd145afa {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 954.139443] env[63273]: WARNING nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Received unexpected event network-vif-plugged-8cd0733e-3bbf-4c4a-bc63-b486fd145afa for instance with vm_state building and task_state spawning. [ 954.139644] env[63273]: DEBUG nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Received event network-changed-8cd0733e-3bbf-4c4a-bc63-b486fd145afa {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 954.139806] env[63273]: DEBUG nova.compute.manager [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Refreshing instance network info cache due to event network-changed-8cd0733e-3bbf-4c4a-bc63-b486fd145afa. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 954.139992] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Acquiring lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 954.140210] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Acquired lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 954.140286] env[63273]: DEBUG nova.network.neutron [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Refreshing network info cache for port 8cd0733e-3bbf-4c4a-bc63-b486fd145afa {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 954.438612] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "13511c60-50cd-44ed-969e-c5fc29b0125e" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 954.448083] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 954.448083] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 954.448188] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 954.512444] env[63273]: DEBUG nova.network.neutron [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Updated VIF entry in instance network info cache for port 8cd0733e-3bbf-4c4a-bc63-b486fd145afa. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 954.512908] env[63273]: DEBUG nova.network.neutron [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Updating instance_info_cache with network_info: [{"id": "2364fdc6-13f8-4243-a4d5-93298a9b5d99", "address": "fa:16:3e:8f:eb:90", "network": {"id": "190ec16f-ddf0-40c9-8de4-e7fb6daa6ee5", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-608361111", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.137", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f3c995e9-7f2f-420c-880a-d60da6e708ad", "external-id": "nsx-vlan-transportzone-166", "segmentation_id": 166, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap2364fdc6-13", "ovs_interfaceid": "2364fdc6-13f8-4243-a4d5-93298a9b5d99", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "9ea385da-a15d-4101-bcc3-0fe11cc3d6c3", "address": "fa:16:3e:88:23:d4", "network": {"id": "b2e91981-edca-4bc0-b508-9bc10e5e9e36", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-793876214", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.136", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dd7d0d95-6848-4e69-ac21-75f8db82a3b5", "external-id": "nsx-vlan-transportzone-272", "segmentation_id": 272, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9ea385da-a1", "ovs_interfaceid": "9ea385da-a15d-4101-bcc3-0fe11cc3d6c3", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "8cd0733e-3bbf-4c4a-bc63-b486fd145afa", "address": "fa:16:3e:88:f6:29", "network": {"id": "190ec16f-ddf0-40c9-8de4-e7fb6daa6ee5", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-608361111", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.248", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f3c995e9-7f2f-420c-880a-d60da6e708ad", "external-id": "nsx-vlan-transportzone-166", "segmentation_id": 166, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8cd0733e-3b", "ovs_interfaceid": "8cd0733e-3bbf-4c4a-bc63-b486fd145afa", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 954.523380] env[63273]: DEBUG oslo_concurrency.lockutils [req-450756aa-43c5-4289-8232-6f66f952bcd5 req-e4a5c2fc-0908-453b-b6e7-9824f2860547 service nova] Releasing lock "refresh_cache-13511c60-50cd-44ed-969e-c5fc29b0125e" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 964.890970] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquiring lock "fefdf558-7a73-4bae-b57c-b86963189ddb" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 964.891323] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Lock "fefdf558-7a73-4bae-b57c-b86963189ddb" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 970.191473] env[63273]: DEBUG oslo_concurrency.lockutils [None req-aa0b3fd2-ad90-475a-891b-dbe52dcb4f62 tempest-ServerShowV257Test-1698833534 tempest-ServerShowV257Test-1698833534-project-member] Acquiring lock "b8bb414b-3759-41d4-8355-4d34cbad4957" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 970.191473] env[63273]: DEBUG oslo_concurrency.lockutils [None req-aa0b3fd2-ad90-475a-891b-dbe52dcb4f62 tempest-ServerShowV257Test-1698833534 tempest-ServerShowV257Test-1698833534-project-member] Lock "b8bb414b-3759-41d4-8355-4d34cbad4957" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 975.909444] env[63273]: DEBUG oslo_concurrency.lockutils [None req-e832ae55-ac39-4c48-83d3-96c5a85e2846 tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquiring lock "083cde6b-b47a-496c-b0e6-4620cba59b9a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 975.909444] env[63273]: DEBUG oslo_concurrency.lockutils [None req-e832ae55-ac39-4c48-83d3-96c5a85e2846 tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "083cde6b-b47a-496c-b0e6-4620cba59b9a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 979.072654] env[63273]: DEBUG oslo_concurrency.lockutils [None req-97db9751-b513-4160-8343-9072eebed2e2 tempest-ServerGroupTestJSON-323358918 tempest-ServerGroupTestJSON-323358918-project-member] Acquiring lock "a31087c9-979d-48dd-975f-322e97f4bbe1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 979.073022] env[63273]: DEBUG oslo_concurrency.lockutils [None req-97db9751-b513-4160-8343-9072eebed2e2 tempest-ServerGroupTestJSON-323358918 tempest-ServerGroupTestJSON-323358918-project-member] Lock "a31087c9-979d-48dd-975f-322e97f4bbe1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 980.891617] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 983.891594] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 984.892548] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 984.908225] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 984.908225] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 984.908225] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 984.908225] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 984.908814] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38133908-58f6-4c16-b806-28e38d07b5b0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.919485] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4ab68d4-5ddc-4d67-95f1-cad1b7d11938 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.938678] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09a34e7b-3536-47d1-865d-d20d71340687 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.949249] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bec12358-1268-468a-bb08-63760bc7be5d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.988163] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180488MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 984.988382] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 984.988633] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 985.089587] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a5c37718-0067-4f5e-ba7e-533f13529739 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 985.089855] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 94d19d7f-8eb7-4c93-aff4-3b1404338240 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 985.090035] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5501c192-867b-40c0-9bea-c3c44865d2f4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 985.090163] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77493b32-817c-4be8-a42d-8d48e707d41f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 985.090345] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 985.090492] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 985.090615] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 985.090731] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e36789b5-f814-4105-b144-361fef9e0d0e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 985.090848] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 985.090961] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 13511c60-50cd-44ed-969e-c5fc29b0125e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 985.109832] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 3e73331f-ce47-4310-98dc-b1a9ac168d30 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.122563] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f5edcebb-61c8-47f5-a925-fd3f0b7a1389 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.135750] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.149354] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.164409] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5b3974c2-5be8-4983-a9d4-962c17889e57 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.176348] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7ed225d1-db66-42c1-99f3-9f3ef4576cfb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.190023] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7476adfd-033e-4b40-899d-1cfd67fd927e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.202305] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 69983659-211d-42be-bb6b-f2ae08db0c9e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.215243] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e6dff48e-986b-422c-82da-532a49b9ae36 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.233103] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fb03832c-8618-40f5-98f1-349b2ca3327e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.255022] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a838375d-aa1b-4e7b-9d17-404d95365b7e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.266862] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c63e222c-c3f7-4543-8d99-e636f240b531 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.280151] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.293722] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b2b1ee46-307f-40fa-8346-394a39a0a99b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.307436] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fefdf558-7a73-4bae-b57c-b86963189ddb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.340857] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b8bb414b-3759-41d4-8355-4d34cbad4957 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.359805] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 083cde6b-b47a-496c-b0e6-4620cba59b9a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.373497] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a31087c9-979d-48dd-975f-322e97f4bbe1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 985.373673] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 985.373837] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '30', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '10', 'num_os_type_None': '10', 'num_proj_d806b4c5831b457fa075e387f55d8ad5': '1', 'io_workload': '10', 'num_proj_4b844f0a784d4f20a5adbf7354c75cc7': '1', 'num_proj_ad03061e544f4b17abb83d9596c461a0': '1', 'num_proj_06e0618687eb4144b9aacdc90a42879d': '1', 'num_proj_70158b9660d14db684be2828e94f8e42': '1', 'num_proj_f1ef99990442451e927a36cdbb924a86': '1', 'num_proj_8189e2aa9960429596f8a65a91032bed': '1', 'num_proj_bc297a27a74f4303b258579a0b803d81': '1', 'num_proj_6a04825b924a482994b31b7066af5714': '1', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 985.882520] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36496c40-ad5e-47ca-8ffa-c3320eeb5708 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.891595] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70bf7a38-5488-439d-9674-3d48a8bb4692 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.933425] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6cfa320c-8ac8-4600-969c-e849bd066f8c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.942301] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cee27b4c-dc85-4ce6-9e9c-a401519bab86 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.956963] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 985.972210] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 985.990857] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 985.991048] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.002s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 986.993339] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 987.028164] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 987.028164] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 988.312833] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c9384bb7-a584-42d6-958b-cd0df3cc5a61 tempest-ServerActionsTestOtherB-1802666129 tempest-ServerActionsTestOtherB-1802666129-project-member] Acquiring lock "75cfbe32-87f1-49ac-9c6a-4e8838e0971b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 988.313168] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c9384bb7-a584-42d6-958b-cd0df3cc5a61 tempest-ServerActionsTestOtherB-1802666129 tempest-ServerActionsTestOtherB-1802666129-project-member] Lock "75cfbe32-87f1-49ac-9c6a-4e8838e0971b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 988.891506] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 988.891692] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 988.891808] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 988.915906] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 988.916073] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 988.916203] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 988.916324] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 988.916443] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 988.916558] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 988.916714] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 988.916906] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 988.917063] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 988.917185] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 988.917303] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 988.917835] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 989.892232] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 989.892487] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 989.892633] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 991.477118] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Acquiring lock "99faf6a2-c9a5-43fc-b7b5-9f8c50253278" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 991.477118] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Lock "99faf6a2-c9a5-43fc-b7b5-9f8c50253278" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 993.370489] env[63273]: WARNING oslo_vmware.rw_handles [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 993.370489] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 993.370489] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 993.370489] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 993.370489] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 993.370489] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 993.370489] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 993.370489] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 993.370489] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 993.370489] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 993.370489] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 993.370489] env[63273]: ERROR oslo_vmware.rw_handles [ 993.371657] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/a743bf18-d208-48ad-94c5-f363167e93b5/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 993.373070] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 993.373329] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Copying Virtual Disk [datastore1] vmware_temp/a743bf18-d208-48ad-94c5-f363167e93b5/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/a743bf18-d208-48ad-94c5-f363167e93b5/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 993.373612] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-a42654fb-8b8c-4923-8d1c-2a673eb26163 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 993.385189] env[63273]: DEBUG oslo_vmware.api [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Waiting for the task: (returnval){ [ 993.385189] env[63273]: value = "task-5072060" [ 993.385189] env[63273]: _type = "Task" [ 993.385189] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 993.394651] env[63273]: DEBUG oslo_vmware.api [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Task: {'id': task-5072060, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 993.896213] env[63273]: DEBUG oslo_vmware.exceptions [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 993.896576] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 993.897204] env[63273]: ERROR nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 993.897204] env[63273]: Faults: ['InvalidArgument'] [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Traceback (most recent call last): [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] yield resources [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] self.driver.spawn(context, instance, image_meta, [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] self._vmops.spawn(context, instance, image_meta, injected_files, [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] self._fetch_image_if_missing(context, vi) [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] image_cache(vi, tmp_image_ds_loc) [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] vm_util.copy_virtual_disk( [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] session._wait_for_task(vmdk_copy_task) [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] return self.wait_for_task(task_ref) [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] return evt.wait() [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] result = hub.switch() [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] return self.greenlet.switch() [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] self.f(*self.args, **self.kw) [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] raise exceptions.translate_fault(task_info.error) [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Faults: ['InvalidArgument'] [ 993.897204] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] [ 993.898093] env[63273]: INFO nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Terminating instance [ 993.899654] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 993.899906] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 993.900575] env[63273]: DEBUG nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 993.900806] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 993.901083] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8784148f-711a-4a4f-80ec-e6f0c2716bb2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 993.903498] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-377ab3ed-0ddb-4687-b737-3ea523e78684 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 993.911096] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 993.913951] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-2d7ef086-e2ba-4639-b7ca-2661576415f0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 993.916933] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 993.917129] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 993.917812] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-3506a514-518c-4a1b-92f0-07e27e709168 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 993.928206] env[63273]: DEBUG oslo_vmware.api [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Waiting for the task: (returnval){ [ 993.928206] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52c2d566-b114-5232-3cf2-ad908c81844b" [ 993.928206] env[63273]: _type = "Task" [ 993.928206] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 993.938605] env[63273]: DEBUG oslo_vmware.api [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52c2d566-b114-5232-3cf2-ad908c81844b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 994.004300] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 994.004530] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 994.004727] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Deleting the datastore file [datastore1] a5c37718-0067-4f5e-ba7e-533f13529739 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 994.005098] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-ffa439a8-0cde-4365-b5c9-7423197c7b0f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.019458] env[63273]: DEBUG oslo_vmware.api [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Waiting for the task: (returnval){ [ 994.019458] env[63273]: value = "task-5072062" [ 994.019458] env[63273]: _type = "Task" [ 994.019458] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 994.029200] env[63273]: DEBUG oslo_vmware.api [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Task: {'id': task-5072062, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 994.447126] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 994.447540] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Creating directory with path [datastore1] vmware_temp/f7186a01-f50b-4597-8989-105faae4df27/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 994.447901] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0b3e4e18-90fa-42d8-934e-14fea30923af {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.464406] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Created directory with path [datastore1] vmware_temp/f7186a01-f50b-4597-8989-105faae4df27/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 994.464775] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Fetch image to [datastore1] vmware_temp/f7186a01-f50b-4597-8989-105faae4df27/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 994.465072] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/f7186a01-f50b-4597-8989-105faae4df27/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 994.466531] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-985aa719-9a33-49f4-bd5a-b7f79d43e644 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.478693] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-289ad018-5385-400a-ae07-3482a9ec8ce2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.490987] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53ca7b54-cd9e-474d-abe0-75f5cd81e3e0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.532101] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e38d6097-add3-45c1-b4bf-dc4009f24c53 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 995.260942] env[63273]: DEBUG oslo_vmware.api [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Task: {'id': task-5072062, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.074526} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 995.262680] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 995.263159] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 995.263159] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 995.263262] env[63273]: INFO nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Took 1.36 seconds to destroy the instance on the hypervisor. [ 995.265454] env[63273]: DEBUG nova.compute.claims [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 995.265626] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 995.266115] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 995.268834] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-ecb53941-e245-40ed-8785-1677bef8b633 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 995.296854] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 995.381781] env[63273]: DEBUG oslo_vmware.rw_handles [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f7186a01-f50b-4597-8989-105faae4df27/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 995.447375] env[63273]: DEBUG oslo_vmware.rw_handles [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 995.447880] env[63273]: DEBUG oslo_vmware.rw_handles [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f7186a01-f50b-4597-8989-105faae4df27/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 995.860287] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d7eb83e-fb17-4dd9-a55c-4e07baa9c03d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 995.869040] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8087a76f-a6ef-48e9-8bab-50ed3f7e1553 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 995.899607] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bea04ff5-16b8-4336-9ea5-865cbe103a9d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 995.907915] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19350d4d-5ae3-4b5c-8818-1af95dbbc662 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 995.922501] env[63273]: DEBUG nova.compute.provider_tree [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 995.940355] env[63273]: DEBUG nova.scheduler.client.report [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 995.960692] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.695s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 995.961134] env[63273]: ERROR nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 995.961134] env[63273]: Faults: ['InvalidArgument'] [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Traceback (most recent call last): [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] self.driver.spawn(context, instance, image_meta, [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] self._vmops.spawn(context, instance, image_meta, injected_files, [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] self._fetch_image_if_missing(context, vi) [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] image_cache(vi, tmp_image_ds_loc) [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] vm_util.copy_virtual_disk( [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] session._wait_for_task(vmdk_copy_task) [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] return self.wait_for_task(task_ref) [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] return evt.wait() [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] result = hub.switch() [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] return self.greenlet.switch() [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] self.f(*self.args, **self.kw) [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] raise exceptions.translate_fault(task_info.error) [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Faults: ['InvalidArgument'] [ 995.961134] env[63273]: ERROR nova.compute.manager [instance: a5c37718-0067-4f5e-ba7e-533f13529739] [ 995.962186] env[63273]: DEBUG nova.compute.utils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 995.963438] env[63273]: DEBUG nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Build of instance a5c37718-0067-4f5e-ba7e-533f13529739 was re-scheduled: A specified parameter was not correct: fileType [ 995.963438] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 995.963816] env[63273]: DEBUG nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 995.963991] env[63273]: DEBUG nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 995.964183] env[63273]: DEBUG nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 995.964378] env[63273]: DEBUG nova.network.neutron [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 996.818854] env[63273]: DEBUG nova.network.neutron [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 996.834894] env[63273]: INFO nova.compute.manager [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Took 0.87 seconds to deallocate network for instance. [ 996.970065] env[63273]: INFO nova.scheduler.client.report [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Deleted allocations for instance a5c37718-0067-4f5e-ba7e-533f13529739 [ 997.001048] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ffae3551-a132-4ae4-9133-a1681f6d0c8e tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Lock "a5c37718-0067-4f5e-ba7e-533f13529739" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 439.924s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 997.003487] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "a5c37718-0067-4f5e-ba7e-533f13529739" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 432.725s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 997.003487] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] During sync_power_state the instance has a pending task (spawning). Skip. [ 997.003487] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "a5c37718-0067-4f5e-ba7e-533f13529739" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 997.003487] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Lock "a5c37718-0067-4f5e-ba7e-533f13529739" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 237.344s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 997.004749] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Acquiring lock "a5c37718-0067-4f5e-ba7e-533f13529739-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 997.005063] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Lock "a5c37718-0067-4f5e-ba7e-533f13529739-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 997.005288] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Lock "a5c37718-0067-4f5e-ba7e-533f13529739-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 997.009127] env[63273]: INFO nova.compute.manager [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Terminating instance [ 997.016195] env[63273]: DEBUG nova.compute.manager [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 997.016195] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 997.016195] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-534537a6-4488-4a54-bb82-6c17bb21d83d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 997.019197] env[63273]: DEBUG nova.compute.manager [None req-32435d1f-c79a-481f-a3ae-3f3f54739371 tempest-ServersTestFqdnHostnames-1404605347 tempest-ServersTestFqdnHostnames-1404605347-project-member] [instance: 2eb7836e-f1ce-4eea-9cee-f9075cdf6d99] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 997.034060] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd83cbf3-f36b-4fe1-a536-7f7a803bf5b5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 997.072139] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a5c37718-0067-4f5e-ba7e-533f13529739 could not be found. [ 997.072500] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 997.072716] env[63273]: INFO nova.compute.manager [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Took 0.06 seconds to destroy the instance on the hypervisor. [ 997.072992] env[63273]: DEBUG oslo.service.loopingcall [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 997.074035] env[63273]: DEBUG nova.compute.manager [-] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 997.074127] env[63273]: DEBUG nova.network.neutron [-] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 997.076520] env[63273]: DEBUG nova.compute.manager [None req-32435d1f-c79a-481f-a3ae-3f3f54739371 tempest-ServersTestFqdnHostnames-1404605347 tempest-ServersTestFqdnHostnames-1404605347-project-member] [instance: 2eb7836e-f1ce-4eea-9cee-f9075cdf6d99] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 997.122447] env[63273]: DEBUG nova.network.neutron [-] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 997.131276] env[63273]: DEBUG oslo_concurrency.lockutils [None req-32435d1f-c79a-481f-a3ae-3f3f54739371 tempest-ServersTestFqdnHostnames-1404605347 tempest-ServersTestFqdnHostnames-1404605347-project-member] Lock "2eb7836e-f1ce-4eea-9cee-f9075cdf6d99" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 214.468s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 997.134916] env[63273]: INFO nova.compute.manager [-] [instance: a5c37718-0067-4f5e-ba7e-533f13529739] Took 0.06 seconds to deallocate network for instance. [ 997.148770] env[63273]: DEBUG nova.compute.manager [None req-7fb717e5-dff6-4664-80ce-1dd824c73301 tempest-ServerAddressesNegativeTestJSON-375425406 tempest-ServerAddressesNegativeTestJSON-375425406-project-member] [instance: 3e73331f-ce47-4310-98dc-b1a9ac168d30] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 997.191491] env[63273]: DEBUG nova.compute.manager [None req-7fb717e5-dff6-4664-80ce-1dd824c73301 tempest-ServerAddressesNegativeTestJSON-375425406 tempest-ServerAddressesNegativeTestJSON-375425406-project-member] [instance: 3e73331f-ce47-4310-98dc-b1a9ac168d30] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 997.222849] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7fb717e5-dff6-4664-80ce-1dd824c73301 tempest-ServerAddressesNegativeTestJSON-375425406 tempest-ServerAddressesNegativeTestJSON-375425406-project-member] Lock "3e73331f-ce47-4310-98dc-b1a9ac168d30" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 208.508s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 997.238801] env[63273]: DEBUG nova.compute.manager [None req-a4cbe97e-784e-4d13-9107-28eb1982fa47 tempest-InstanceActionsV221TestJSON-243712533 tempest-InstanceActionsV221TestJSON-243712533-project-member] [instance: f5edcebb-61c8-47f5-a925-fd3f0b7a1389] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 997.284027] env[63273]: DEBUG nova.compute.manager [None req-a4cbe97e-784e-4d13-9107-28eb1982fa47 tempest-InstanceActionsV221TestJSON-243712533 tempest-InstanceActionsV221TestJSON-243712533-project-member] [instance: f5edcebb-61c8-47f5-a925-fd3f0b7a1389] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 997.315029] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a0f09e3c-e483-49e5-884d-48576de817e4 tempest-FloatingIPsAssociationTestJSON-2000280200 tempest-FloatingIPsAssociationTestJSON-2000280200-project-member] Lock "a5c37718-0067-4f5e-ba7e-533f13529739" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.311s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 997.330706] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4cbe97e-784e-4d13-9107-28eb1982fa47 tempest-InstanceActionsV221TestJSON-243712533 tempest-InstanceActionsV221TestJSON-243712533-project-member] Lock "f5edcebb-61c8-47f5-a925-fd3f0b7a1389" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 200.813s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 997.348352] env[63273]: DEBUG nova.compute.manager [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 997.447237] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 997.447511] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 997.452031] env[63273]: INFO nova.compute.claims [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 997.956378] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Acquiring lock "7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.024747] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f4c7d37-dd8c-4893-abbb-c02298871c05 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.033860] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1e67476-1c5c-4924-a3c2-17b734657b9c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.065527] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd43cfd6-83bc-428a-8e10-c3a3a977ab8f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.074399] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c818f75-be8e-4b3e-8d5d-f7c9d7be5e6b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.097869] env[63273]: DEBUG nova.compute.provider_tree [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 998.108959] env[63273]: DEBUG nova.scheduler.client.report [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 998.128789] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.681s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.129346] env[63273]: DEBUG nova.compute.manager [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 998.187547] env[63273]: DEBUG nova.compute.claims [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 998.187808] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.188074] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.680972] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22b4c012-30ff-4d31-a019-4f1fa1af358f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.693770] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f6d841c-4a3e-447a-be81-deac05fff70c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.726687] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b54b828-38e1-481f-9436-ad870b77672b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.735149] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e2eb499-e75b-49c4-a24c-7e76a1a98837 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.750334] env[63273]: DEBUG nova.compute.provider_tree [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 998.762253] env[63273]: DEBUG nova.scheduler.client.report [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 998.786165] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.598s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.786958] env[63273]: DEBUG nova.compute.utils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Conflict updating instance 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca. Expected: {'task_state': [None]}. Actual: {'task_state': 'deleting'} {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 998.789061] env[63273]: DEBUG nova.compute.manager [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Instance disappeared during build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2505}} [ 998.789061] env[63273]: DEBUG nova.compute.manager [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 998.789061] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Acquiring lock "refresh_cache-7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 998.789061] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Acquired lock "refresh_cache-7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 998.789257] env[63273]: DEBUG nova.network.neutron [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 998.817886] env[63273]: DEBUG nova.network.neutron [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 999.005902] env[63273]: DEBUG nova.network.neutron [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 999.016769] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Releasing lock "refresh_cache-7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 999.018102] env[63273]: DEBUG nova.compute.manager [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 999.018102] env[63273]: DEBUG nova.compute.manager [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 999.018102] env[63273]: DEBUG nova.network.neutron [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 999.042960] env[63273]: DEBUG nova.network.neutron [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 999.056351] env[63273]: DEBUG nova.network.neutron [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 999.068586] env[63273]: INFO nova.compute.manager [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Took 0.05 seconds to deallocate network for instance. [ 999.157484] env[63273]: INFO nova.scheduler.client.report [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Deleted allocations for instance 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca [ 999.157795] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0c9d9416-58e2-4adf-ab9c-23c265c39f7c tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Lock "7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 199.451s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 999.159061] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Lock "7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 1.203s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 999.159291] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Acquiring lock "7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 999.159669] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Lock "7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 999.159887] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Lock "7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 999.164098] env[63273]: INFO nova.compute.manager [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Terminating instance [ 999.166597] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Acquiring lock "refresh_cache-7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 999.166914] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Acquired lock "refresh_cache-7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 999.167116] env[63273]: DEBUG nova.network.neutron [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 999.173190] env[63273]: DEBUG nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 999.197488] env[63273]: DEBUG nova.network.neutron [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 999.228758] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 999.229095] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 999.230636] env[63273]: INFO nova.compute.claims [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 999.328944] env[63273]: DEBUG nova.network.neutron [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 999.339191] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Releasing lock "refresh_cache-7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 999.339612] env[63273]: DEBUG nova.compute.manager [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 999.339810] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 999.340369] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d039790e-63a6-4344-bf97-36bdabe4e424 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.356421] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c43e0cf7-630e-4a06-9678-94a25f19d45f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.394872] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca could not be found. [ 999.394872] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 999.394996] env[63273]: INFO nova.compute.manager [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Took 0.06 seconds to destroy the instance on the hypervisor. [ 999.395235] env[63273]: DEBUG oslo.service.loopingcall [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 999.395467] env[63273]: DEBUG nova.compute.manager [-] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 999.395559] env[63273]: DEBUG nova.network.neutron [-] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 999.428568] env[63273]: DEBUG nova.network.neutron [-] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 999.440339] env[63273]: DEBUG nova.network.neutron [-] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 999.454183] env[63273]: INFO nova.compute.manager [-] [instance: 7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca] Took 0.06 seconds to deallocate network for instance. [ 999.585989] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ee81b955-1314-43e1-bd21-e407f93a50eb tempest-ServerMetadataTestJSON-993462663 tempest-ServerMetadataTestJSON-993462663-project-member] Lock "7c6ae4a7-b2b0-4ff5-945d-68ee3e9191ca" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.427s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 999.754117] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee3ec905-2477-4916-9cec-508e4085e14d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.765414] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44bc5431-7d77-4988-8029-592be023bf6a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.796635] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c95c619d-21d1-4568-8b97-976edcb1314a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.809922] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d99ecb5c-06be-481e-987f-361221718db4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.821446] env[63273]: DEBUG nova.compute.provider_tree [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 999.835041] env[63273]: DEBUG nova.scheduler.client.report [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 999.850425] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.621s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 999.852465] env[63273]: DEBUG nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 999.892432] env[63273]: DEBUG nova.compute.utils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 999.892432] env[63273]: DEBUG nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Not allocating networking since 'none' was specified. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1969}} [ 999.904278] env[63273]: DEBUG nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 999.983104] env[63273]: DEBUG nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1000.017260] env[63273]: DEBUG nova.virt.hardware [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1000.017645] env[63273]: DEBUG nova.virt.hardware [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1000.017645] env[63273]: DEBUG nova.virt.hardware [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1000.017645] env[63273]: DEBUG nova.virt.hardware [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1000.017924] env[63273]: DEBUG nova.virt.hardware [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1000.017924] env[63273]: DEBUG nova.virt.hardware [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1000.018454] env[63273]: DEBUG nova.virt.hardware [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1000.018694] env[63273]: DEBUG nova.virt.hardware [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1000.018922] env[63273]: DEBUG nova.virt.hardware [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1000.019282] env[63273]: DEBUG nova.virt.hardware [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1000.019561] env[63273]: DEBUG nova.virt.hardware [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1000.021210] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b75607b-e202-4065-b10f-c103f52f385d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.030561] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-723f2a0b-6120-4196-b25e-ca6525e59a47 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.046344] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Instance VIF info [] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1000.053930] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Creating folder: Project (ff52e0a10f1c464f808b6df84185058f). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1000.054985] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-064f74e5-284b-4e6e-81bb-b31ec4450e88 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.066313] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Created folder: Project (ff52e0a10f1c464f808b6df84185058f) in parent group-v986930. [ 1000.066514] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Creating folder: Instances. Parent ref: group-v986987. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1000.066765] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-27a6afe8-cc3c-4bd0-9268-1e4be69f4fa8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.078391] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Created folder: Instances in parent group-v986987. [ 1000.078391] env[63273]: DEBUG oslo.service.loopingcall [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1000.078391] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1000.078391] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-e71923ec-954a-46b6-882f-946ab9fce015 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.097407] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1000.097407] env[63273]: value = "task-5072065" [ 1000.097407] env[63273]: _type = "Task" [ 1000.097407] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1000.108258] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072065, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1000.608606] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072065, 'name': CreateVM_Task, 'duration_secs': 0.257911} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1000.608606] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1000.608885] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1000.609049] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1000.609377] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1000.609620] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c3ed53c5-8504-438f-bfef-776a08937b3f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.614965] env[63273]: DEBUG oslo_vmware.api [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Waiting for the task: (returnval){ [ 1000.614965] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]521ad498-a377-755d-e619-a7004147a9ec" [ 1000.614965] env[63273]: _type = "Task" [ 1000.614965] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1000.626063] env[63273]: DEBUG oslo_vmware.api [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]521ad498-a377-755d-e619-a7004147a9ec, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1001.130076] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1001.131353] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1001.131956] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1001.541601] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquiring lock "a081322d-4636-4bfc-90f6-bd0c617a09e8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1001.541601] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Lock "a081322d-4636-4bfc-90f6-bd0c617a09e8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1006.408539] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquiring lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1007.279130] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6ef1ec96-65a8-4255-842d-368764dc3f04 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "5d370386-d8e7-401e-8fc3-c181d74d8125" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1007.279516] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6ef1ec96-65a8-4255-842d-368764dc3f04 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "5d370386-d8e7-401e-8fc3-c181d74d8125" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1010.760398] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f6d49f4a-4f3e-4553-b1aa-a53e13e531a7 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquiring lock "f65fc3d9-407a-47bd-a038-211b25931d38" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1010.760652] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f6d49f4a-4f3e-4553-b1aa-a53e13e531a7 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "f65fc3d9-407a-47bd-a038-211b25931d38" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1011.547509] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b0c8c1fe-cc6d-442f-bde4-9fff100444d6 tempest-AttachVolumeTestJSON-293439156 tempest-AttachVolumeTestJSON-293439156-project-member] Acquiring lock "77786a5e-1cb6-4100-a2fc-e67e0140e7ca" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1011.547509] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b0c8c1fe-cc6d-442f-bde4-9fff100444d6 tempest-AttachVolumeTestJSON-293439156 tempest-AttachVolumeTestJSON-293439156-project-member] Lock "77786a5e-1cb6-4100-a2fc-e67e0140e7ca" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1017.099699] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a792310a-0dc8-49f4-b3f4-9d8aba6bc69e tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] Acquiring lock "5d7dbf9b-1ab3-459a-ad02-451f2c522584" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1017.099699] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a792310a-0dc8-49f4-b3f4-9d8aba6bc69e tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] Lock "5d7dbf9b-1ab3-459a-ad02-451f2c522584" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1040.871469] env[63273]: WARNING oslo_vmware.rw_handles [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1040.871469] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1040.871469] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1040.871469] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1040.871469] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1040.871469] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1040.871469] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1040.871469] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1040.871469] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1040.871469] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1040.871469] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1040.871469] env[63273]: ERROR oslo_vmware.rw_handles [ 1040.872086] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/f7186a01-f50b-4597-8989-105faae4df27/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1040.873697] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1040.874427] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Copying Virtual Disk [datastore1] vmware_temp/f7186a01-f50b-4597-8989-105faae4df27/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/f7186a01-f50b-4597-8989-105faae4df27/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1040.874867] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-5a4a81a5-85bc-48d9-a04a-f4f6b9d7cdfb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1040.884804] env[63273]: DEBUG oslo_vmware.api [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Waiting for the task: (returnval){ [ 1040.884804] env[63273]: value = "task-5072066" [ 1040.884804] env[63273]: _type = "Task" [ 1040.884804] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1040.893929] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1040.894208] env[63273]: DEBUG oslo_vmware.api [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Task: {'id': task-5072066, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1041.396037] env[63273]: DEBUG oslo_vmware.exceptions [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1041.396037] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1041.396464] env[63273]: ERROR nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1041.396464] env[63273]: Faults: ['InvalidArgument'] [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Traceback (most recent call last): [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] yield resources [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] self.driver.spawn(context, instance, image_meta, [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] self._fetch_image_if_missing(context, vi) [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] image_cache(vi, tmp_image_ds_loc) [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] vm_util.copy_virtual_disk( [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] session._wait_for_task(vmdk_copy_task) [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] return self.wait_for_task(task_ref) [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] return evt.wait() [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] result = hub.switch() [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] return self.greenlet.switch() [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] self.f(*self.args, **self.kw) [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] raise exceptions.translate_fault(task_info.error) [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Faults: ['InvalidArgument'] [ 1041.396464] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] [ 1041.397336] env[63273]: INFO nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Terminating instance [ 1041.398412] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1041.398619] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1041.398878] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-54cc520f-6d31-4ba0-a6ed-69f532e97538 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.401251] env[63273]: DEBUG nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1041.401396] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1041.402124] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8824bc0-015c-4626-8ffe-e51291897baa {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.410997] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1041.412135] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-012e7d51-ccd4-456d-a034-1a84bdcbf21e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.413859] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1041.414049] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1041.414720] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-bcacde06-632f-44a3-b157-092836ba9de8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.420579] env[63273]: DEBUG oslo_vmware.api [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Waiting for the task: (returnval){ [ 1041.420579] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e8327a-3d5d-3d3b-69a5-6b15b69a9bb3" [ 1041.420579] env[63273]: _type = "Task" [ 1041.420579] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1041.430339] env[63273]: DEBUG oslo_vmware.api [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e8327a-3d5d-3d3b-69a5-6b15b69a9bb3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1041.481515] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1041.481881] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1041.482108] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Deleting the datastore file [datastore1] 94d19d7f-8eb7-4c93-aff4-3b1404338240 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1041.482408] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-87fcbc34-0690-4b01-83c7-1a175db99cfb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.492811] env[63273]: DEBUG oslo_vmware.api [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Waiting for the task: (returnval){ [ 1041.492811] env[63273]: value = "task-5072068" [ 1041.492811] env[63273]: _type = "Task" [ 1041.492811] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1041.504665] env[63273]: DEBUG oslo_vmware.api [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Task: {'id': task-5072068, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1041.930753] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1041.931041] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Creating directory with path [datastore1] vmware_temp/71a2232e-ee0c-40d6-bff0-4a4eff59708d/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1041.931281] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-9d03fc0f-7b8d-4205-b069-024d906b3ee7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.944207] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Created directory with path [datastore1] vmware_temp/71a2232e-ee0c-40d6-bff0-4a4eff59708d/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1041.944349] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Fetch image to [datastore1] vmware_temp/71a2232e-ee0c-40d6-bff0-4a4eff59708d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1041.944518] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/71a2232e-ee0c-40d6-bff0-4a4eff59708d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1041.945352] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc15475b-88d6-406c-b916-8d4572eaf400 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.952052] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78a0dc9e-6acf-4a63-8066-eb55868f74f7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.960951] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3aad4285-cd4c-4014-8c1c-f1a288a71e15 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.990663] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1bb1f83a-df11-4442-93bd-9fe420a6a361 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.998114] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-7505977f-bea9-4c29-ae2f-087d4e925dc9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.002417] env[63273]: DEBUG oslo_vmware.api [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Task: {'id': task-5072068, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.08284} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1042.002970] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1042.003202] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1042.003425] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1042.003602] env[63273]: INFO nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1042.005797] env[63273]: DEBUG nova.compute.claims [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1042.006016] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1042.006263] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1042.094104] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1042.152219] env[63273]: DEBUG oslo_vmware.rw_handles [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/71a2232e-ee0c-40d6-bff0-4a4eff59708d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1042.212375] env[63273]: DEBUG oslo_vmware.rw_handles [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1042.212590] env[63273]: DEBUG oslo_vmware.rw_handles [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/71a2232e-ee0c-40d6-bff0-4a4eff59708d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1042.427262] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7052eb7-2ba0-41c6-8499-5b22a3a98b07 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.436190] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07d65bf2-91a2-42a5-aab8-4bb626e9d364 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.467992] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34252d5d-f039-45eb-960d-dcccef747318 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.476491] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64117868-87e5-4bae-b2e6-1297431fef3e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.491738] env[63273]: DEBUG nova.compute.provider_tree [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1042.503108] env[63273]: DEBUG nova.scheduler.client.report [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1042.519652] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.512s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1042.519652] env[63273]: ERROR nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1042.519652] env[63273]: Faults: ['InvalidArgument'] [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Traceback (most recent call last): [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] self.driver.spawn(context, instance, image_meta, [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] self._fetch_image_if_missing(context, vi) [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] image_cache(vi, tmp_image_ds_loc) [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] vm_util.copy_virtual_disk( [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] session._wait_for_task(vmdk_copy_task) [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] return self.wait_for_task(task_ref) [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] return evt.wait() [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] result = hub.switch() [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] return self.greenlet.switch() [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] self.f(*self.args, **self.kw) [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] raise exceptions.translate_fault(task_info.error) [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Faults: ['InvalidArgument'] [ 1042.519652] env[63273]: ERROR nova.compute.manager [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] [ 1042.520829] env[63273]: DEBUG nova.compute.utils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1042.521845] env[63273]: DEBUG nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Build of instance 94d19d7f-8eb7-4c93-aff4-3b1404338240 was re-scheduled: A specified parameter was not correct: fileType [ 1042.521845] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1042.522314] env[63273]: DEBUG nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1042.522562] env[63273]: DEBUG nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1042.522754] env[63273]: DEBUG nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1042.522916] env[63273]: DEBUG nova.network.neutron [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1043.108150] env[63273]: DEBUG nova.network.neutron [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1043.122282] env[63273]: INFO nova.compute.manager [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Took 0.60 seconds to deallocate network for instance. [ 1043.231647] env[63273]: INFO nova.scheduler.client.report [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Deleted allocations for instance 94d19d7f-8eb7-4c93-aff4-3b1404338240 [ 1043.253811] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6362bfe3-fb7e-4e65-9c5a-a80641f80852 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Lock "94d19d7f-8eb7-4c93-aff4-3b1404338240" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 481.127s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.255954] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "94d19d7f-8eb7-4c93-aff4-3b1404338240" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 478.978s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1043.255954] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] During sync_power_state the instance has a pending task (spawning). Skip. [ 1043.255954] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "94d19d7f-8eb7-4c93-aff4-3b1404338240" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.256314] env[63273]: DEBUG oslo_concurrency.lockutils [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Lock "94d19d7f-8eb7-4c93-aff4-3b1404338240" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 280.837s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1043.256655] env[63273]: DEBUG oslo_concurrency.lockutils [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Acquiring lock "94d19d7f-8eb7-4c93-aff4-3b1404338240-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1043.256955] env[63273]: DEBUG oslo_concurrency.lockutils [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Lock "94d19d7f-8eb7-4c93-aff4-3b1404338240-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1043.257242] env[63273]: DEBUG oslo_concurrency.lockutils [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Lock "94d19d7f-8eb7-4c93-aff4-3b1404338240-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.259614] env[63273]: INFO nova.compute.manager [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Terminating instance [ 1043.262513] env[63273]: DEBUG nova.compute.manager [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1043.262513] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1043.262513] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ae3e103d-fd6a-4d5b-8093-f7382a1cc557 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1043.273823] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f473cca-2587-4583-9786-368cd25bb721 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1043.284145] env[63273]: DEBUG nova.compute.manager [None req-7eda3871-e07e-4a99-a7d9-54904f21d085 tempest-AttachVolumeTestJSON-293439156 tempest-AttachVolumeTestJSON-293439156-project-member] [instance: 5b3974c2-5be8-4983-a9d4-962c17889e57] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1043.309687] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 94d19d7f-8eb7-4c93-aff4-3b1404338240 could not be found. [ 1043.309923] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1043.310115] env[63273]: INFO nova.compute.manager [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1043.310360] env[63273]: DEBUG oslo.service.loopingcall [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1043.310598] env[63273]: DEBUG nova.compute.manager [-] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1043.310694] env[63273]: DEBUG nova.network.neutron [-] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1043.313899] env[63273]: DEBUG nova.compute.manager [None req-7eda3871-e07e-4a99-a7d9-54904f21d085 tempest-AttachVolumeTestJSON-293439156 tempest-AttachVolumeTestJSON-293439156-project-member] [instance: 5b3974c2-5be8-4983-a9d4-962c17889e57] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1043.336130] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7eda3871-e07e-4a99-a7d9-54904f21d085 tempest-AttachVolumeTestJSON-293439156 tempest-AttachVolumeTestJSON-293439156-project-member] Lock "5b3974c2-5be8-4983-a9d4-962c17889e57" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 231.072s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.339278] env[63273]: DEBUG nova.network.neutron [-] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1043.346048] env[63273]: INFO nova.compute.manager [-] [instance: 94d19d7f-8eb7-4c93-aff4-3b1404338240] Took 0.03 seconds to deallocate network for instance. [ 1043.350751] env[63273]: DEBUG nova.compute.manager [None req-92e7eb84-c0c9-44d2-a93b-07f36b8d8135 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 7ed225d1-db66-42c1-99f3-9f3ef4576cfb] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1043.376957] env[63273]: DEBUG nova.compute.manager [None req-92e7eb84-c0c9-44d2-a93b-07f36b8d8135 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 7ed225d1-db66-42c1-99f3-9f3ef4576cfb] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1043.396430] env[63273]: DEBUG oslo_concurrency.lockutils [None req-92e7eb84-c0c9-44d2-a93b-07f36b8d8135 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "7ed225d1-db66-42c1-99f3-9f3ef4576cfb" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 229.669s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.410059] env[63273]: DEBUG nova.compute.manager [None req-b1ca5531-4fc3-45c9-8d37-3c468e0e32d3 tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] [instance: 7476adfd-033e-4b40-899d-1cfd67fd927e] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1043.440017] env[63273]: DEBUG nova.compute.manager [None req-b1ca5531-4fc3-45c9-8d37-3c468e0e32d3 tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] [instance: 7476adfd-033e-4b40-899d-1cfd67fd927e] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1043.446967] env[63273]: DEBUG oslo_concurrency.lockutils [None req-12cd1f05-90fc-4335-bfc7-a0326c5fcfb6 tempest-FloatingIPsAssociationNegativeTestJSON-767083702 tempest-FloatingIPsAssociationNegativeTestJSON-767083702-project-member] Lock "94d19d7f-8eb7-4c93-aff4-3b1404338240" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.191s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.461726] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b1ca5531-4fc3-45c9-8d37-3c468e0e32d3 tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] Lock "7476adfd-033e-4b40-899d-1cfd67fd927e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 224.939s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.474423] env[63273]: DEBUG nova.compute.manager [None req-fa64e833-ace5-44b5-bb8d-5cd192a1e920 tempest-ServerActionsTestOtherA-2021597575 tempest-ServerActionsTestOtherA-2021597575-project-member] [instance: 69983659-211d-42be-bb6b-f2ae08db0c9e] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1043.503088] env[63273]: DEBUG nova.compute.manager [None req-fa64e833-ace5-44b5-bb8d-5cd192a1e920 tempest-ServerActionsTestOtherA-2021597575 tempest-ServerActionsTestOtherA-2021597575-project-member] [instance: 69983659-211d-42be-bb6b-f2ae08db0c9e] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1043.524268] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fa64e833-ace5-44b5-bb8d-5cd192a1e920 tempest-ServerActionsTestOtherA-2021597575 tempest-ServerActionsTestOtherA-2021597575-project-member] Lock "69983659-211d-42be-bb6b-f2ae08db0c9e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 218.912s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.533061] env[63273]: DEBUG nova.compute.manager [None req-e54ff226-ad2b-4659-ae44-20fc0e22e0e4 tempest-ServerRescueNegativeTestJSON-1207392049 tempest-ServerRescueNegativeTestJSON-1207392049-project-member] [instance: e6dff48e-986b-422c-82da-532a49b9ae36] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1043.556074] env[63273]: DEBUG nova.compute.manager [None req-e54ff226-ad2b-4659-ae44-20fc0e22e0e4 tempest-ServerRescueNegativeTestJSON-1207392049 tempest-ServerRescueNegativeTestJSON-1207392049-project-member] [instance: e6dff48e-986b-422c-82da-532a49b9ae36] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1043.576727] env[63273]: DEBUG oslo_concurrency.lockutils [None req-e54ff226-ad2b-4659-ae44-20fc0e22e0e4 tempest-ServerRescueNegativeTestJSON-1207392049 tempest-ServerRescueNegativeTestJSON-1207392049-project-member] Lock "e6dff48e-986b-422c-82da-532a49b9ae36" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 216.479s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.585246] env[63273]: DEBUG nova.compute.manager [None req-1f8d4d1a-fa4c-4247-88a2-bcb870e5009b tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] [instance: fb03832c-8618-40f5-98f1-349b2ca3327e] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1043.609881] env[63273]: DEBUG nova.compute.manager [None req-1f8d4d1a-fa4c-4247-88a2-bcb870e5009b tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] [instance: fb03832c-8618-40f5-98f1-349b2ca3327e] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1043.630737] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1f8d4d1a-fa4c-4247-88a2-bcb870e5009b tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Lock "fb03832c-8618-40f5-98f1-349b2ca3327e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 216.361s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.640207] env[63273]: DEBUG nova.compute.manager [None req-1f8d4d1a-fa4c-4247-88a2-bcb870e5009b tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] [instance: a838375d-aa1b-4e7b-9d17-404d95365b7e] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1043.664829] env[63273]: DEBUG nova.compute.manager [None req-1f8d4d1a-fa4c-4247-88a2-bcb870e5009b tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] [instance: a838375d-aa1b-4e7b-9d17-404d95365b7e] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1043.685054] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1f8d4d1a-fa4c-4247-88a2-bcb870e5009b tempest-MultipleCreateTestJSON-2035323882 tempest-MultipleCreateTestJSON-2035323882-project-member] Lock "a838375d-aa1b-4e7b-9d17-404d95365b7e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 216.376s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.693380] env[63273]: DEBUG nova.compute.manager [None req-36b44b09-5dfb-4bfa-9deb-4666c11732b7 tempest-ServerRescueNegativeTestJSON-1207392049 tempest-ServerRescueNegativeTestJSON-1207392049-project-member] [instance: c63e222c-c3f7-4543-8d99-e636f240b531] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1043.719030] env[63273]: DEBUG nova.compute.manager [None req-36b44b09-5dfb-4bfa-9deb-4666c11732b7 tempest-ServerRescueNegativeTestJSON-1207392049 tempest-ServerRescueNegativeTestJSON-1207392049-project-member] [instance: c63e222c-c3f7-4543-8d99-e636f240b531] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1043.765035] env[63273]: DEBUG oslo_concurrency.lockutils [None req-36b44b09-5dfb-4bfa-9deb-4666c11732b7 tempest-ServerRescueNegativeTestJSON-1207392049 tempest-ServerRescueNegativeTestJSON-1207392049-project-member] Lock "c63e222c-c3f7-4543-8d99-e636f240b531" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 215.134s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1043.776352] env[63273]: DEBUG nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1043.833771] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1043.834038] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1043.835652] env[63273]: INFO nova.compute.claims [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1043.890809] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1044.136184] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7239e39b-a1e8-426c-b951-43b5f7398a70 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1044.144069] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff1c1892-ce05-4ea6-a4cf-a8994eab8d4a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1044.173239] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd81da8b-b0d6-4a91-bbdc-65d9934875f3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1044.180638] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-546ab6fc-944c-4e6c-9a8b-7e03ad7dc4dc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1044.193492] env[63273]: DEBUG nova.compute.provider_tree [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1044.202579] env[63273]: DEBUG nova.scheduler.client.report [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1044.217349] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.383s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1044.217844] env[63273]: DEBUG nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1044.251252] env[63273]: DEBUG nova.compute.utils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1044.253888] env[63273]: DEBUG nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1044.254133] env[63273]: DEBUG nova.network.neutron [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1044.262793] env[63273]: DEBUG nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1044.325768] env[63273]: DEBUG nova.policy [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7648b8a75224424f8226b574da373a87', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '495ae26325184812a744fc95da67038b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1044.334829] env[63273]: DEBUG nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1044.360364] env[63273]: DEBUG nova.virt.hardware [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1044.360613] env[63273]: DEBUG nova.virt.hardware [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1044.360769] env[63273]: DEBUG nova.virt.hardware [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1044.360961] env[63273]: DEBUG nova.virt.hardware [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1044.361101] env[63273]: DEBUG nova.virt.hardware [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1044.361251] env[63273]: DEBUG nova.virt.hardware [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1044.361457] env[63273]: DEBUG nova.virt.hardware [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1044.361614] env[63273]: DEBUG nova.virt.hardware [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1044.361778] env[63273]: DEBUG nova.virt.hardware [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1044.362113] env[63273]: DEBUG nova.virt.hardware [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1044.362205] env[63273]: DEBUG nova.virt.hardware [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1044.362995] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42c665f0-fde7-4a45-af60-b8a0529dc622 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1044.372400] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e5ea0fe-f976-4629-bb59-347cdaad4865 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1044.745880] env[63273]: DEBUG nova.network.neutron [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Successfully created port: 4007d751-43a8-4dd0-8627-fd90dd8f877c {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1045.429966] env[63273]: DEBUG nova.compute.manager [req-7bea2e55-271f-4173-a0b7-8affb0d7b394 req-3fcb65c5-19bc-4c46-892b-6e33b6919818 service nova] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Received event network-vif-plugged-4007d751-43a8-4dd0-8627-fd90dd8f877c {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1045.430275] env[63273]: DEBUG oslo_concurrency.lockutils [req-7bea2e55-271f-4173-a0b7-8affb0d7b394 req-3fcb65c5-19bc-4c46-892b-6e33b6919818 service nova] Acquiring lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1045.430489] env[63273]: DEBUG oslo_concurrency.lockutils [req-7bea2e55-271f-4173-a0b7-8affb0d7b394 req-3fcb65c5-19bc-4c46-892b-6e33b6919818 service nova] Lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1045.430641] env[63273]: DEBUG oslo_concurrency.lockutils [req-7bea2e55-271f-4173-a0b7-8affb0d7b394 req-3fcb65c5-19bc-4c46-892b-6e33b6919818 service nova] Lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1045.430751] env[63273]: DEBUG nova.compute.manager [req-7bea2e55-271f-4173-a0b7-8affb0d7b394 req-3fcb65c5-19bc-4c46-892b-6e33b6919818 service nova] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] No waiting events found dispatching network-vif-plugged-4007d751-43a8-4dd0-8627-fd90dd8f877c {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1045.430911] env[63273]: WARNING nova.compute.manager [req-7bea2e55-271f-4173-a0b7-8affb0d7b394 req-3fcb65c5-19bc-4c46-892b-6e33b6919818 service nova] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Received unexpected event network-vif-plugged-4007d751-43a8-4dd0-8627-fd90dd8f877c for instance with vm_state building and task_state spawning. [ 1045.541118] env[63273]: DEBUG nova.network.neutron [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Successfully updated port: 4007d751-43a8-4dd0-8627-fd90dd8f877c {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1045.558232] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "refresh_cache-0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1045.558393] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "refresh_cache-0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1045.560713] env[63273]: DEBUG nova.network.neutron [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1045.612230] env[63273]: DEBUG nova.network.neutron [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1045.818099] env[63273]: DEBUG nova.network.neutron [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Updating instance_info_cache with network_info: [{"id": "4007d751-43a8-4dd0-8627-fd90dd8f877c", "address": "fa:16:3e:d0:ea:c0", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4007d751-43", "ovs_interfaceid": "4007d751-43a8-4dd0-8627-fd90dd8f877c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1045.830973] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "refresh_cache-0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1045.831289] env[63273]: DEBUG nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Instance network_info: |[{"id": "4007d751-43a8-4dd0-8627-fd90dd8f877c", "address": "fa:16:3e:d0:ea:c0", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4007d751-43", "ovs_interfaceid": "4007d751-43a8-4dd0-8627-fd90dd8f877c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1045.832033] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:d0:ea:c0', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '69e1a5c1-4ae7-409b-8de7-d401684ef60d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '4007d751-43a8-4dd0-8627-fd90dd8f877c', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1045.839119] env[63273]: DEBUG oslo.service.loopingcall [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1045.839599] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1045.839832] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-b368b235-3363-4c98-b485-52045634fdef {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1045.859786] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1045.859786] env[63273]: value = "task-5072069" [ 1045.859786] env[63273]: _type = "Task" [ 1045.859786] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1045.867734] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072069, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1046.370422] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072069, 'name': CreateVM_Task, 'duration_secs': 0.319102} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1046.370608] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1046.371318] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1046.371548] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1046.371901] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1046.372177] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f7466a0b-402d-4695-891a-192800e3b741 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1046.377440] env[63273]: DEBUG oslo_vmware.api [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 1046.377440] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52b586f9-9d69-47cc-4d5c-cd33f660fd61" [ 1046.377440] env[63273]: _type = "Task" [ 1046.377440] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1046.387657] env[63273]: DEBUG oslo_vmware.api [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52b586f9-9d69-47cc-4d5c-cd33f660fd61, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1046.890117] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1046.890492] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1046.890565] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1046.890944] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1046.902568] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1046.902789] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1046.902962] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1046.903134] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1046.904333] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6cbd7194-61ad-4be8-8063-4aea7fb8e3f6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1046.913566] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1af9a181-0139-4644-b411-9ff80fbff390 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1046.929592] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f52e0c98-359b-4a02-bc4a-d726d20b592a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1046.936991] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7de3924f-554b-48b3-b2ab-9f5f55a08aee {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1046.967026] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180485MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1046.967264] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1046.967400] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1047.052359] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5501c192-867b-40c0-9bea-c3c44865d2f4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1047.052530] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77493b32-817c-4be8-a42d-8d48e707d41f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1047.052683] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1047.052832] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1047.052957] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1047.053090] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e36789b5-f814-4105-b144-361fef9e0d0e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1047.053246] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1047.053361] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 13511c60-50cd-44ed-969e-c5fc29b0125e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1047.053477] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1047.053591] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1047.065393] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b2b1ee46-307f-40fa-8346-394a39a0a99b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.078322] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fefdf558-7a73-4bae-b57c-b86963189ddb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.090170] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b8bb414b-3759-41d4-8355-4d34cbad4957 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.104691] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 083cde6b-b47a-496c-b0e6-4620cba59b9a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.116445] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a31087c9-979d-48dd-975f-322e97f4bbe1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.130213] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 75cfbe32-87f1-49ac-9c6a-4e8838e0971b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.141943] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 99faf6a2-c9a5-43fc-b7b5-9f8c50253278 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.154601] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a081322d-4636-4bfc-90f6-bd0c617a09e8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.166801] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d370386-d8e7-401e-8fc3-c181d74d8125 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.177507] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f65fc3d9-407a-47bd-a038-211b25931d38 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.188547] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77786a5e-1cb6-4100-a2fc-e67e0140e7ca has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.202636] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d7dbf9b-1ab3-459a-ad02-451f2c522584 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1047.202891] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1047.203071] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '44', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '9', 'num_os_type_None': '10', 'num_proj_ad03061e544f4b17abb83d9596c461a0': '1', 'io_workload': '10', 'num_proj_06e0618687eb4144b9aacdc90a42879d': '1', 'num_proj_70158b9660d14db684be2828e94f8e42': '1', 'num_proj_f1ef99990442451e927a36cdbb924a86': '1', 'num_proj_8189e2aa9960429596f8a65a91032bed': '1', 'num_proj_bc297a27a74f4303b258579a0b803d81': '1', 'num_proj_6a04825b924a482994b31b7066af5714': '1', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1', 'num_proj_ff52e0a10f1c464f808b6df84185058f': '1', 'num_task_spawning': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1047.457946] env[63273]: DEBUG nova.compute.manager [req-2a924c83-c975-4a0d-b348-66864444a247 req-ddb9a6e6-2f29-447d-940b-e49aecb9494e service nova] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Received event network-changed-4007d751-43a8-4dd0-8627-fd90dd8f877c {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1047.458244] env[63273]: DEBUG nova.compute.manager [req-2a924c83-c975-4a0d-b348-66864444a247 req-ddb9a6e6-2f29-447d-940b-e49aecb9494e service nova] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Refreshing instance network info cache due to event network-changed-4007d751-43a8-4dd0-8627-fd90dd8f877c. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1047.458489] env[63273]: DEBUG oslo_concurrency.lockutils [req-2a924c83-c975-4a0d-b348-66864444a247 req-ddb9a6e6-2f29-447d-940b-e49aecb9494e service nova] Acquiring lock "refresh_cache-0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1047.458688] env[63273]: DEBUG oslo_concurrency.lockutils [req-2a924c83-c975-4a0d-b348-66864444a247 req-ddb9a6e6-2f29-447d-940b-e49aecb9494e service nova] Acquired lock "refresh_cache-0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1047.458854] env[63273]: DEBUG nova.network.neutron [req-2a924c83-c975-4a0d-b348-66864444a247 req-ddb9a6e6-2f29-447d-940b-e49aecb9494e service nova] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Refreshing network info cache for port 4007d751-43a8-4dd0-8627-fd90dd8f877c {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1047.510473] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81091f17-5579-46ca-ad1f-618823f9ab59 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.519218] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd3974f5-b5af-454b-8fb1-8f867d2331c8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.552420] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0338900f-2b4f-4aa2-837e-8988359c1573 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.561324] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e0cbd60-2e92-48f7-a08c-589da5fee122 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.575449] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1047.584820] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1047.606442] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1047.606626] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.639s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1047.833606] env[63273]: DEBUG nova.network.neutron [req-2a924c83-c975-4a0d-b348-66864444a247 req-ddb9a6e6-2f29-447d-940b-e49aecb9494e service nova] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Updated VIF entry in instance network info cache for port 4007d751-43a8-4dd0-8627-fd90dd8f877c. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1047.833976] env[63273]: DEBUG nova.network.neutron [req-2a924c83-c975-4a0d-b348-66864444a247 req-ddb9a6e6-2f29-447d-940b-e49aecb9494e service nova] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Updating instance_info_cache with network_info: [{"id": "4007d751-43a8-4dd0-8627-fd90dd8f877c", "address": "fa:16:3e:d0:ea:c0", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4007d751-43", "ovs_interfaceid": "4007d751-43a8-4dd0-8627-fd90dd8f877c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1047.845643] env[63273]: DEBUG oslo_concurrency.lockutils [req-2a924c83-c975-4a0d-b348-66864444a247 req-ddb9a6e6-2f29-447d-940b-e49aecb9494e service nova] Releasing lock "refresh_cache-0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1048.606795] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1048.607073] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1048.892067] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1048.892067] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1048.892229] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1048.913984] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1048.914164] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1048.914300] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1048.914428] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1048.914553] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1048.914673] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1048.914792] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1048.914911] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1048.915040] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1048.915162] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1048.915281] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1049.891934] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1049.891934] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1050.887816] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1050.891474] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1088.005337] env[63273]: WARNING oslo_vmware.rw_handles [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1088.005337] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1088.005337] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1088.005337] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1088.005337] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1088.005337] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1088.005337] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1088.005337] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1088.005337] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1088.005337] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1088.005337] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1088.005337] env[63273]: ERROR oslo_vmware.rw_handles [ 1088.006269] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/71a2232e-ee0c-40d6-bff0-4a4eff59708d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1088.007600] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1088.007805] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Copying Virtual Disk [datastore1] vmware_temp/71a2232e-ee0c-40d6-bff0-4a4eff59708d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/71a2232e-ee0c-40d6-bff0-4a4eff59708d/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1088.008109] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-1572b9e2-2467-48dd-ab27-b1de55b316e2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1088.016704] env[63273]: DEBUG oslo_vmware.api [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Waiting for the task: (returnval){ [ 1088.016704] env[63273]: value = "task-5072070" [ 1088.016704] env[63273]: _type = "Task" [ 1088.016704] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1088.025250] env[63273]: DEBUG oslo_vmware.api [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Task: {'id': task-5072070, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1088.527556] env[63273]: DEBUG oslo_vmware.exceptions [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1088.527842] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1088.528427] env[63273]: ERROR nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1088.528427] env[63273]: Faults: ['InvalidArgument'] [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Traceback (most recent call last): [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] yield resources [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] self.driver.spawn(context, instance, image_meta, [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] self._fetch_image_if_missing(context, vi) [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] image_cache(vi, tmp_image_ds_loc) [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] vm_util.copy_virtual_disk( [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] session._wait_for_task(vmdk_copy_task) [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] return self.wait_for_task(task_ref) [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] return evt.wait() [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] result = hub.switch() [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] return self.greenlet.switch() [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] self.f(*self.args, **self.kw) [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] raise exceptions.translate_fault(task_info.error) [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Faults: ['InvalidArgument'] [ 1088.528427] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] [ 1088.529458] env[63273]: INFO nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Terminating instance [ 1088.530804] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1088.531128] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1088.531981] env[63273]: DEBUG nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1088.532299] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1088.532627] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-55e56596-37e0-4675-adb0-1cbc3b6e5421 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1088.535340] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b4105bc-4457-432d-8608-f6bb882dc10b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1088.543044] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1088.543304] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-e6b1f052-df4e-4add-b772-68e9dd75f6a9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1088.545741] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1088.545936] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1088.546985] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b6b07989-dbfa-47cc-a32f-dc32b4acbea8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1088.552338] env[63273]: DEBUG oslo_vmware.api [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Waiting for the task: (returnval){ [ 1088.552338] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52b3400d-adab-4d42-f433-0144eee52cba" [ 1088.552338] env[63273]: _type = "Task" [ 1088.552338] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1088.561620] env[63273]: DEBUG oslo_vmware.api [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52b3400d-adab-4d42-f433-0144eee52cba, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1088.621017] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1088.621250] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1088.621428] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Deleting the datastore file [datastore1] 5501c192-867b-40c0-9bea-c3c44865d2f4 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1088.621707] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-a9879c90-84eb-4891-b7c4-dc981a1fc5eb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1088.629601] env[63273]: DEBUG oslo_vmware.api [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Waiting for the task: (returnval){ [ 1088.629601] env[63273]: value = "task-5072072" [ 1088.629601] env[63273]: _type = "Task" [ 1088.629601] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1088.638025] env[63273]: DEBUG oslo_vmware.api [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Task: {'id': task-5072072, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1089.063986] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1089.065163] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Creating directory with path [datastore1] vmware_temp/4f3b98a2-54e9-4fd2-b127-628d2864400a/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1089.065163] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-778c94af-1eed-42c7-bad4-0775814a82ee {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1089.078791] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Created directory with path [datastore1] vmware_temp/4f3b98a2-54e9-4fd2-b127-628d2864400a/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1089.078791] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Fetch image to [datastore1] vmware_temp/4f3b98a2-54e9-4fd2-b127-628d2864400a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1089.078791] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/4f3b98a2-54e9-4fd2-b127-628d2864400a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1089.078791] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-704a7b55-255d-40ea-8e0c-0184ad23ae17 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1089.085915] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18d85267-f2ff-4cda-83c2-497852766292 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1089.095528] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aeb4ed1a-6f9a-4805-8f1b-d5303fe06c34 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1089.127578] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90a1524c-5192-4af4-a76d-11fb1206febc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1089.140048] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-57b74f9f-d463-412f-a637-11bc30b63c8c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1089.141123] env[63273]: DEBUG oslo_vmware.api [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Task: {'id': task-5072072, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.081283} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1089.141363] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1089.141551] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1089.141711] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1089.141883] env[63273]: INFO nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Took 0.61 seconds to destroy the instance on the hypervisor. [ 1089.144058] env[63273]: DEBUG nova.compute.claims [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1089.144237] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1089.144447] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1089.167938] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1089.232371] env[63273]: DEBUG oslo_vmware.rw_handles [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4f3b98a2-54e9-4fd2-b127-628d2864400a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1089.295572] env[63273]: DEBUG oslo_vmware.rw_handles [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1089.295777] env[63273]: DEBUG oslo_vmware.rw_handles [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4f3b98a2-54e9-4fd2-b127-628d2864400a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1089.528722] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6b2734f-61b4-479a-b77f-46d8cd195211 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1089.536909] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de41814b-a6c3-4e6e-938d-f6e9713da7c2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1089.566352] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53579683-c57f-432f-bfe9-97c0fe3b152f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1089.574225] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3398e21d-b29a-4614-941d-6991df4deca9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1089.589047] env[63273]: DEBUG nova.compute.provider_tree [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1089.602082] env[63273]: DEBUG nova.scheduler.client.report [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1089.619547] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.475s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1089.620094] env[63273]: ERROR nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1089.620094] env[63273]: Faults: ['InvalidArgument'] [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Traceback (most recent call last): [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] self.driver.spawn(context, instance, image_meta, [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] self._fetch_image_if_missing(context, vi) [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] image_cache(vi, tmp_image_ds_loc) [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] vm_util.copy_virtual_disk( [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] session._wait_for_task(vmdk_copy_task) [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] return self.wait_for_task(task_ref) [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] return evt.wait() [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] result = hub.switch() [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] return self.greenlet.switch() [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] self.f(*self.args, **self.kw) [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] raise exceptions.translate_fault(task_info.error) [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Faults: ['InvalidArgument'] [ 1089.620094] env[63273]: ERROR nova.compute.manager [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] [ 1089.621113] env[63273]: DEBUG nova.compute.utils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1089.622335] env[63273]: DEBUG nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Build of instance 5501c192-867b-40c0-9bea-c3c44865d2f4 was re-scheduled: A specified parameter was not correct: fileType [ 1089.622335] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1089.622704] env[63273]: DEBUG nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1089.622878] env[63273]: DEBUG nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1089.623031] env[63273]: DEBUG nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1089.623195] env[63273]: DEBUG nova.network.neutron [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1090.311649] env[63273]: DEBUG nova.network.neutron [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1090.332484] env[63273]: INFO nova.compute.manager [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Took 0.71 seconds to deallocate network for instance. [ 1090.438346] env[63273]: INFO nova.scheduler.client.report [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Deleted allocations for instance 5501c192-867b-40c0-9bea-c3c44865d2f4 [ 1090.460788] env[63273]: DEBUG oslo_concurrency.lockutils [None req-802a7155-a056-4976-aaa0-06791b65392f tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Lock "5501c192-867b-40c0-9bea-c3c44865d2f4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 525.686s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1090.462149] env[63273]: DEBUG oslo_concurrency.lockutils [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Lock "5501c192-867b-40c0-9bea-c3c44865d2f4" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 327.515s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1090.462355] env[63273]: DEBUG oslo_concurrency.lockutils [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Acquiring lock "5501c192-867b-40c0-9bea-c3c44865d2f4-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1090.462559] env[63273]: DEBUG oslo_concurrency.lockutils [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Lock "5501c192-867b-40c0-9bea-c3c44865d2f4-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1090.462775] env[63273]: DEBUG oslo_concurrency.lockutils [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Lock "5501c192-867b-40c0-9bea-c3c44865d2f4-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1090.466523] env[63273]: INFO nova.compute.manager [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Terminating instance [ 1090.468772] env[63273]: DEBUG nova.compute.manager [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1090.469146] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1090.470393] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-267dbb70-cb0d-4e8b-9894-737ac0e1a8cf {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1090.476154] env[63273]: DEBUG nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1090.483986] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b46a122-cac3-4166-8fa7-f99498eb8d7e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1090.517549] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5501c192-867b-40c0-9bea-c3c44865d2f4 could not be found. [ 1090.517775] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1090.517963] env[63273]: INFO nova.compute.manager [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1090.518914] env[63273]: DEBUG oslo.service.loopingcall [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1090.523141] env[63273]: DEBUG nova.compute.manager [-] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1090.523236] env[63273]: DEBUG nova.network.neutron [-] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1090.537424] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1090.537663] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1090.539163] env[63273]: INFO nova.compute.claims [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1090.555989] env[63273]: DEBUG nova.network.neutron [-] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1090.584468] env[63273]: INFO nova.compute.manager [-] [instance: 5501c192-867b-40c0-9bea-c3c44865d2f4] Took 0.06 seconds to deallocate network for instance. [ 1090.679269] env[63273]: DEBUG oslo_concurrency.lockutils [None req-536c80ca-876d-448b-8da1-fd99d82a9514 tempest-ServersAdminNegativeTestJSON-2023174960 tempest-ServersAdminNegativeTestJSON-2023174960-project-member] Lock "5501c192-867b-40c0-9bea-c3c44865d2f4" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.217s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1090.862748] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e206018d-aff2-40fd-87e4-60e8ba6e651e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1090.872053] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20f8f938-54db-40e2-9d9c-f14085ae7db2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1090.903138] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-256e0fcd-a61a-4e0c-bdde-647bcd3c3271 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1090.911634] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd31044a-abe7-4c3c-b050-ddce9258b8ec {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1090.925248] env[63273]: DEBUG nova.compute.provider_tree [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1090.934244] env[63273]: DEBUG nova.scheduler.client.report [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1090.948939] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.411s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1090.949495] env[63273]: DEBUG nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1090.988217] env[63273]: DEBUG nova.compute.utils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1090.990073] env[63273]: DEBUG nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1090.990073] env[63273]: DEBUG nova.network.neutron [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1091.001263] env[63273]: DEBUG nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1091.071290] env[63273]: DEBUG nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1091.084163] env[63273]: DEBUG nova.policy [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ba6c85eb49a749bbb96e8cc474f206bd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '418e6c5db17d4baf9c4f7cbc229ad07b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1091.096465] env[63273]: DEBUG nova.virt.hardware [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:11:22Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='464185ed-89d2-42d1-9336-9ec95cd81be7',id=38,is_public=True,memory_mb=128,name='tempest-test_resize_flavor_-2036812731',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1091.096695] env[63273]: DEBUG nova.virt.hardware [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1091.096851] env[63273]: DEBUG nova.virt.hardware [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1091.097060] env[63273]: DEBUG nova.virt.hardware [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1091.097189] env[63273]: DEBUG nova.virt.hardware [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1091.097337] env[63273]: DEBUG nova.virt.hardware [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1091.097543] env[63273]: DEBUG nova.virt.hardware [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1091.097703] env[63273]: DEBUG nova.virt.hardware [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1091.097870] env[63273]: DEBUG nova.virt.hardware [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1091.098045] env[63273]: DEBUG nova.virt.hardware [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1091.098291] env[63273]: DEBUG nova.virt.hardware [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1091.099172] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a041e433-f968-40cf-84e5-2d393b4a4e0e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1091.107605] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-043e2ee6-676e-437f-99ff-2c77be11cf62 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1091.532432] env[63273]: DEBUG nova.network.neutron [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Successfully created port: 5282c075-04e2-4864-a374-e3b170c9dd88 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1092.188793] env[63273]: DEBUG nova.network.neutron [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Successfully updated port: 5282c075-04e2-4864-a374-e3b170c9dd88 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1092.200352] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "refresh_cache-b2b1ee46-307f-40fa-8346-394a39a0a99b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1092.200497] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquired lock "refresh_cache-b2b1ee46-307f-40fa-8346-394a39a0a99b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1092.200685] env[63273]: DEBUG nova.network.neutron [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1092.266151] env[63273]: DEBUG nova.network.neutron [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1092.469955] env[63273]: DEBUG nova.compute.manager [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Received event network-vif-plugged-5282c075-04e2-4864-a374-e3b170c9dd88 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1092.470511] env[63273]: DEBUG oslo_concurrency.lockutils [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] Acquiring lock "b2b1ee46-307f-40fa-8346-394a39a0a99b-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1092.470792] env[63273]: DEBUG oslo_concurrency.lockutils [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] Lock "b2b1ee46-307f-40fa-8346-394a39a0a99b-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1092.471011] env[63273]: DEBUG oslo_concurrency.lockutils [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] Lock "b2b1ee46-307f-40fa-8346-394a39a0a99b-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1092.471246] env[63273]: DEBUG nova.compute.manager [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] No waiting events found dispatching network-vif-plugged-5282c075-04e2-4864-a374-e3b170c9dd88 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1092.471520] env[63273]: WARNING nova.compute.manager [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Received unexpected event network-vif-plugged-5282c075-04e2-4864-a374-e3b170c9dd88 for instance with vm_state building and task_state spawning. [ 1092.471706] env[63273]: DEBUG nova.compute.manager [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Received event network-changed-5282c075-04e2-4864-a374-e3b170c9dd88 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1092.471883] env[63273]: DEBUG nova.compute.manager [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Refreshing instance network info cache due to event network-changed-5282c075-04e2-4864-a374-e3b170c9dd88. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1092.472095] env[63273]: DEBUG oslo_concurrency.lockutils [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] Acquiring lock "refresh_cache-b2b1ee46-307f-40fa-8346-394a39a0a99b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1092.520973] env[63273]: DEBUG nova.network.neutron [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Updating instance_info_cache with network_info: [{"id": "5282c075-04e2-4864-a374-e3b170c9dd88", "address": "fa:16:3e:32:f6:5a", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.127", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5282c075-04", "ovs_interfaceid": "5282c075-04e2-4864-a374-e3b170c9dd88", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1092.536386] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Releasing lock "refresh_cache-b2b1ee46-307f-40fa-8346-394a39a0a99b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1092.536702] env[63273]: DEBUG nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Instance network_info: |[{"id": "5282c075-04e2-4864-a374-e3b170c9dd88", "address": "fa:16:3e:32:f6:5a", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.127", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5282c075-04", "ovs_interfaceid": "5282c075-04e2-4864-a374-e3b170c9dd88", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1092.537115] env[63273]: DEBUG oslo_concurrency.lockutils [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] Acquired lock "refresh_cache-b2b1ee46-307f-40fa-8346-394a39a0a99b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1092.537348] env[63273]: DEBUG nova.network.neutron [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Refreshing network info cache for port 5282c075-04e2-4864-a374-e3b170c9dd88 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1092.538547] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:32:f6:5a', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f66f8375-4460-4acd-987b-acda72bfcf0d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '5282c075-04e2-4864-a374-e3b170c9dd88', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1092.546457] env[63273]: DEBUG oslo.service.loopingcall [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1092.547682] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1092.550055] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-ede6f5e0-d11c-4b76-99bf-ad98972c5a21 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1092.571410] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1092.571410] env[63273]: value = "task-5072073" [ 1092.571410] env[63273]: _type = "Task" [ 1092.571410] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1092.580098] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072073, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1092.854869] env[63273]: DEBUG nova.network.neutron [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Updated VIF entry in instance network info cache for port 5282c075-04e2-4864-a374-e3b170c9dd88. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1092.855299] env[63273]: DEBUG nova.network.neutron [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Updating instance_info_cache with network_info: [{"id": "5282c075-04e2-4864-a374-e3b170c9dd88", "address": "fa:16:3e:32:f6:5a", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.127", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5282c075-04", "ovs_interfaceid": "5282c075-04e2-4864-a374-e3b170c9dd88", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1092.865684] env[63273]: DEBUG oslo_concurrency.lockutils [req-958abf61-79cc-4179-99db-ac369fd36bd0 req-4f377182-5ec2-4d59-98e5-e2247769006e service nova] Releasing lock "refresh_cache-b2b1ee46-307f-40fa-8346-394a39a0a99b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1093.081870] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072073, 'name': CreateVM_Task, 'duration_secs': 0.299856} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1093.082064] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1093.082709] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1093.082874] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1093.083217] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1093.083606] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-535e0de4-9f8a-4ef4-bba2-10e5b7b2c6f8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1093.088219] env[63273]: DEBUG oslo_vmware.api [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Waiting for the task: (returnval){ [ 1093.088219] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]528e9a11-f3e8-e0bf-ea1a-f98425adbe90" [ 1093.088219] env[63273]: _type = "Task" [ 1093.088219] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1093.095841] env[63273]: DEBUG oslo_vmware.api [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]528e9a11-f3e8-e0bf-ea1a-f98425adbe90, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1093.599960] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1093.600636] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1093.601028] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1094.587851] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1101.892223] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1102.892541] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1102.892541] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances with incomplete migration {{(pid=63273) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11345}} [ 1103.904289] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1106.454823] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1106.455190] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1106.886946] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1106.956931] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1106.957239] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1107.891209] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1107.891618] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1107.907404] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1107.907649] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1107.907822] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1107.907982] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1107.909464] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e291ab1-bf3e-4f8b-ac31-409557906e8c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1107.918838] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ae2f3a1-92d7-4ea1-a68b-9f37df30ac60 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1107.934550] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e626551b-3235-4d6c-8dd5-a2fde114a0e2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1107.941972] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91a02274-3117-4468-be7f-97cdd88ba9a7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1107.973117] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180500MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1107.973309] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1107.973478] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1108.057881] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77493b32-817c-4be8-a42d-8d48e707d41f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1108.057881] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1108.057881] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1108.057881] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1108.057881] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e36789b5-f814-4105-b144-361fef9e0d0e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1108.057881] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1108.057881] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 13511c60-50cd-44ed-969e-c5fc29b0125e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1108.057881] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1108.058371] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1108.058371] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b2b1ee46-307f-40fa-8346-394a39a0a99b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1108.071144] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fefdf558-7a73-4bae-b57c-b86963189ddb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.084363] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b8bb414b-3759-41d4-8355-4d34cbad4957 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.096119] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 083cde6b-b47a-496c-b0e6-4620cba59b9a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.109028] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a31087c9-979d-48dd-975f-322e97f4bbe1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.120940] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 75cfbe32-87f1-49ac-9c6a-4e8838e0971b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.132323] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 99faf6a2-c9a5-43fc-b7b5-9f8c50253278 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.143243] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a081322d-4636-4bfc-90f6-bd0c617a09e8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.153892] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d370386-d8e7-401e-8fc3-c181d74d8125 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.165714] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f65fc3d9-407a-47bd-a038-211b25931d38 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.177745] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77786a5e-1cb6-4100-a2fc-e67e0140e7ca has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.188378] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d7dbf9b-1ab3-459a-ad02-451f2c522584 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.200023] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.211598] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1108.211839] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1108.212056] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '45', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '9', 'num_os_type_None': '10', 'num_proj_06e0618687eb4144b9aacdc90a42879d': '1', 'io_workload': '10', 'num_proj_70158b9660d14db684be2828e94f8e42': '1', 'num_proj_f1ef99990442451e927a36cdbb924a86': '1', 'num_proj_8189e2aa9960429596f8a65a91032bed': '1', 'num_proj_bc297a27a74f4303b258579a0b803d81': '1', 'num_proj_6a04825b924a482994b31b7066af5714': '1', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1', 'num_proj_ff52e0a10f1c464f808b6df84185058f': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_task_spawning': '1', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1108.510139] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4bee06b-09e7-4ae2-80bb-ff9171e013fc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1108.517960] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a84dbd7a-39d7-49b0-88e4-eaafc4a4cc8c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1108.548126] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2272e23-2e76-4b2b-99c6-b27552b81813 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1108.556108] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53acc5d7-45aa-4958-bf07-4e9b41dcc56e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1108.569959] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1108.580737] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1108.597300] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1108.597493] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.624s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1108.891259] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1108.891554] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1108.891592] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1108.913321] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1108.913551] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1108.913600] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1108.913765] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1108.913905] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1108.914063] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1108.914193] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1108.914329] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1108.914451] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1108.914572] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1108.914693] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1108.915360] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1108.915602] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1109.899640] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1109.899936] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1109.900061] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1109.900197] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11307}} [ 1109.911332] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] There are 0 instances to clean {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11316}} [ 1111.903793] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1112.887918] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1138.416964] env[63273]: WARNING oslo_vmware.rw_handles [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1138.416964] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1138.416964] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1138.416964] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1138.416964] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1138.416964] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1138.416964] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1138.416964] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1138.416964] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1138.416964] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1138.416964] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1138.416964] env[63273]: ERROR oslo_vmware.rw_handles [ 1138.417661] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/4f3b98a2-54e9-4fd2-b127-628d2864400a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1138.419654] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1138.419953] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Copying Virtual Disk [datastore1] vmware_temp/4f3b98a2-54e9-4fd2-b127-628d2864400a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/4f3b98a2-54e9-4fd2-b127-628d2864400a/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1138.420305] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-d469ece6-74ed-4777-9530-7b0f558d4792 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1138.429587] env[63273]: DEBUG oslo_vmware.api [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Waiting for the task: (returnval){ [ 1138.429587] env[63273]: value = "task-5072074" [ 1138.429587] env[63273]: _type = "Task" [ 1138.429587] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1138.438904] env[63273]: DEBUG oslo_vmware.api [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Task: {'id': task-5072074, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1138.940132] env[63273]: DEBUG oslo_vmware.exceptions [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1138.940448] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1138.941109] env[63273]: ERROR nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1138.941109] env[63273]: Faults: ['InvalidArgument'] [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Traceback (most recent call last): [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] yield resources [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] self.driver.spawn(context, instance, image_meta, [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] self._fetch_image_if_missing(context, vi) [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] image_cache(vi, tmp_image_ds_loc) [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] vm_util.copy_virtual_disk( [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] session._wait_for_task(vmdk_copy_task) [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] return self.wait_for_task(task_ref) [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] return evt.wait() [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] result = hub.switch() [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] return self.greenlet.switch() [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] self.f(*self.args, **self.kw) [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] raise exceptions.translate_fault(task_info.error) [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Faults: ['InvalidArgument'] [ 1138.941109] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] [ 1138.942649] env[63273]: INFO nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Terminating instance [ 1138.944250] env[63273]: DEBUG nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1138.944470] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1138.944929] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1138.945162] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1138.945941] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ae78ef7-5c08-4116-9a36-856460508e56 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1138.950026] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-287c7e1f-d621-4d8e-b28f-50df684e4445 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1138.956853] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1138.958110] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-ab7e0d93-8539-45f8-b48d-b4b001208c78 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1138.959698] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1138.959878] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1138.960565] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-4beb1d82-ec76-4be9-9b72-d5c5b5f23e5b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1138.966594] env[63273]: DEBUG oslo_vmware.api [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Waiting for the task: (returnval){ [ 1138.966594] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5225dd60-487b-1cba-88dd-8bab75827e4b" [ 1138.966594] env[63273]: _type = "Task" [ 1138.966594] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1138.974839] env[63273]: DEBUG oslo_vmware.api [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5225dd60-487b-1cba-88dd-8bab75827e4b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1139.037811] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1139.038128] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1139.038319] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Deleting the datastore file [datastore1] 77493b32-817c-4be8-a42d-8d48e707d41f {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1139.038702] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-b179ffa2-8935-46a6-a1a6-e5bca09b8268 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1139.047908] env[63273]: DEBUG oslo_vmware.api [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Waiting for the task: (returnval){ [ 1139.047908] env[63273]: value = "task-5072076" [ 1139.047908] env[63273]: _type = "Task" [ 1139.047908] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1139.058753] env[63273]: DEBUG oslo_vmware.api [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Task: {'id': task-5072076, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1139.477484] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1139.477777] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Creating directory with path [datastore1] vmware_temp/7a12cdea-0edd-4e22-a2ea-8cc24f5c03de/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1139.478055] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8cc8ff0a-5ad4-40f6-b46c-d7f31ef46be1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1139.490833] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Created directory with path [datastore1] vmware_temp/7a12cdea-0edd-4e22-a2ea-8cc24f5c03de/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1139.491057] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Fetch image to [datastore1] vmware_temp/7a12cdea-0edd-4e22-a2ea-8cc24f5c03de/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1139.491235] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/7a12cdea-0edd-4e22-a2ea-8cc24f5c03de/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1139.492078] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd4369bb-6fe1-4b10-a43d-07fa1c4d5eda {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1139.500217] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f365ba7f-54c3-45e6-9c6c-0c3e3cdbd762 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1139.512821] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30292190-d1f2-4f60-9126-a3c7030d1700 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1139.563983] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8faa5672-2cad-406c-be01-5fbfb4d47132 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1139.573039] env[63273]: DEBUG oslo_vmware.api [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Task: {'id': task-5072076, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.080512} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1139.573794] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1139.573987] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1139.574177] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1139.574354] env[63273]: INFO nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Took 0.63 seconds to destroy the instance on the hypervisor. [ 1139.576486] env[63273]: DEBUG nova.compute.claims [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1139.576666] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1139.576877] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1139.579398] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-b688d5c5-6e86-464f-86a0-3b2dcb2c524c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1139.603349] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1139.672506] env[63273]: DEBUG oslo_vmware.rw_handles [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7a12cdea-0edd-4e22-a2ea-8cc24f5c03de/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1139.733999] env[63273]: DEBUG oslo_vmware.rw_handles [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1139.733999] env[63273]: DEBUG oslo_vmware.rw_handles [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7a12cdea-0edd-4e22-a2ea-8cc24f5c03de/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1139.967306] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-325ccf70-4867-4d1c-bec6-128fc84178b6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1139.975489] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eae85bc6-ff08-4851-b6ae-5971dece5fc8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1140.007265] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd961733-ea49-4dd8-b5cd-347e508d38ce {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1140.015383] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18165412-ac48-42c7-8e81-6261220ed1e6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1140.031094] env[63273]: DEBUG nova.compute.provider_tree [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1140.040579] env[63273]: DEBUG nova.scheduler.client.report [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1140.056097] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.479s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1140.056715] env[63273]: ERROR nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1140.056715] env[63273]: Faults: ['InvalidArgument'] [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Traceback (most recent call last): [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] self.driver.spawn(context, instance, image_meta, [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] self._fetch_image_if_missing(context, vi) [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] image_cache(vi, tmp_image_ds_loc) [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] vm_util.copy_virtual_disk( [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] session._wait_for_task(vmdk_copy_task) [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] return self.wait_for_task(task_ref) [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] return evt.wait() [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] result = hub.switch() [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] return self.greenlet.switch() [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] self.f(*self.args, **self.kw) [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] raise exceptions.translate_fault(task_info.error) [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Faults: ['InvalidArgument'] [ 1140.056715] env[63273]: ERROR nova.compute.manager [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] [ 1140.057824] env[63273]: DEBUG nova.compute.utils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1140.059611] env[63273]: DEBUG nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Build of instance 77493b32-817c-4be8-a42d-8d48e707d41f was re-scheduled: A specified parameter was not correct: fileType [ 1140.059611] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1140.059996] env[63273]: DEBUG nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1140.060211] env[63273]: DEBUG nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1140.060452] env[63273]: DEBUG nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1140.060723] env[63273]: DEBUG nova.network.neutron [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1140.460901] env[63273]: DEBUG nova.network.neutron [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1140.484734] env[63273]: INFO nova.compute.manager [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Took 0.42 seconds to deallocate network for instance. [ 1140.589503] env[63273]: INFO nova.scheduler.client.report [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Deleted allocations for instance 77493b32-817c-4be8-a42d-8d48e707d41f [ 1140.613853] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c1596b23-dd39-4242-9af1-e0a2434c3ee2 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Lock "77493b32-817c-4be8-a42d-8d48e707d41f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 573.661s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1140.615039] env[63273]: DEBUG oslo_concurrency.lockutils [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Lock "77493b32-817c-4be8-a42d-8d48e707d41f" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 374.487s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1140.615248] env[63273]: DEBUG oslo_concurrency.lockutils [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Acquiring lock "77493b32-817c-4be8-a42d-8d48e707d41f-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1140.615504] env[63273]: DEBUG oslo_concurrency.lockutils [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Lock "77493b32-817c-4be8-a42d-8d48e707d41f-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1140.615854] env[63273]: DEBUG oslo_concurrency.lockutils [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Lock "77493b32-817c-4be8-a42d-8d48e707d41f-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1140.618289] env[63273]: INFO nova.compute.manager [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Terminating instance [ 1140.620355] env[63273]: DEBUG nova.compute.manager [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1140.620552] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1140.621044] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-467c982c-f5a6-490f-ba8d-4d72bbdc4653 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1140.630663] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15fa4fb2-013e-41dc-a4dc-3d8549d18e85 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1140.642597] env[63273]: DEBUG nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1140.667586] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 77493b32-817c-4be8-a42d-8d48e707d41f could not be found. [ 1140.667586] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1140.667586] env[63273]: INFO nova.compute.manager [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1140.667586] env[63273]: DEBUG oslo.service.loopingcall [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1140.667586] env[63273]: DEBUG nova.compute.manager [-] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1140.667586] env[63273]: DEBUG nova.network.neutron [-] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1140.695335] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1140.695587] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1140.697202] env[63273]: INFO nova.compute.claims [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1140.701728] env[63273]: DEBUG nova.network.neutron [-] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1140.710764] env[63273]: INFO nova.compute.manager [-] [instance: 77493b32-817c-4be8-a42d-8d48e707d41f] Took 0.04 seconds to deallocate network for instance. [ 1140.821929] env[63273]: DEBUG oslo_concurrency.lockutils [None req-531b2ed3-345e-430c-b237-8a6d1ce8e852 tempest-InstanceActionsTestJSON-474597627 tempest-InstanceActionsTestJSON-474597627-project-member] Lock "77493b32-817c-4be8-a42d-8d48e707d41f" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.207s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1141.036427] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65bc6abe-c6d0-4eb4-8f54-a74a1e35b5fb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1141.044296] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b827b691-e07a-4f14-b117-c98efd7f13d9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1141.074697] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac03a483-65f9-4d8a-891a-d5c212c31b34 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1141.083077] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dac22d2e-4b77-4b02-88ae-b5ddd9165cad {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1141.098974] env[63273]: DEBUG nova.compute.provider_tree [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1141.109134] env[63273]: DEBUG nova.scheduler.client.report [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1141.125017] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.429s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1141.125532] env[63273]: DEBUG nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1141.163147] env[63273]: DEBUG nova.compute.utils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1141.164675] env[63273]: DEBUG nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1141.164924] env[63273]: DEBUG nova.network.neutron [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1141.176133] env[63273]: DEBUG nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1141.243356] env[63273]: DEBUG nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1141.260790] env[63273]: DEBUG nova.policy [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6e780dfe43a44055b3d33bab044e79b0', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ad0ab37da3b64969894f8ab378f35ff7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1141.278593] env[63273]: DEBUG nova.virt.hardware [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1141.278843] env[63273]: DEBUG nova.virt.hardware [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1141.279042] env[63273]: DEBUG nova.virt.hardware [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1141.279237] env[63273]: DEBUG nova.virt.hardware [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1141.279386] env[63273]: DEBUG nova.virt.hardware [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1141.279531] env[63273]: DEBUG nova.virt.hardware [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1141.279756] env[63273]: DEBUG nova.virt.hardware [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1141.279927] env[63273]: DEBUG nova.virt.hardware [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1141.280120] env[63273]: DEBUG nova.virt.hardware [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1141.280288] env[63273]: DEBUG nova.virt.hardware [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1141.280459] env[63273]: DEBUG nova.virt.hardware [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1141.281686] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ce0997c-96cf-4faa-8efd-46487e45bfa7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1141.292081] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50c94157-e278-48f2-86b1-b9246fbe2cd7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1141.954201] env[63273]: DEBUG nova.network.neutron [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Successfully created port: e316d37b-4368-453f-b8f6-57baa2a3c503 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1142.641919] env[63273]: DEBUG nova.compute.manager [req-af5b29f0-9d0f-4ba5-bb1d-a0345e1c6af8 req-e8697f8c-5327-407b-87c5-7dadd90c8ed8 service nova] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Received event network-vif-plugged-e316d37b-4368-453f-b8f6-57baa2a3c503 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1142.642177] env[63273]: DEBUG oslo_concurrency.lockutils [req-af5b29f0-9d0f-4ba5-bb1d-a0345e1c6af8 req-e8697f8c-5327-407b-87c5-7dadd90c8ed8 service nova] Acquiring lock "fefdf558-7a73-4bae-b57c-b86963189ddb-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1142.642532] env[63273]: DEBUG oslo_concurrency.lockutils [req-af5b29f0-9d0f-4ba5-bb1d-a0345e1c6af8 req-e8697f8c-5327-407b-87c5-7dadd90c8ed8 service nova] Lock "fefdf558-7a73-4bae-b57c-b86963189ddb-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1142.642654] env[63273]: DEBUG oslo_concurrency.lockutils [req-af5b29f0-9d0f-4ba5-bb1d-a0345e1c6af8 req-e8697f8c-5327-407b-87c5-7dadd90c8ed8 service nova] Lock "fefdf558-7a73-4bae-b57c-b86963189ddb-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1142.642913] env[63273]: DEBUG nova.compute.manager [req-af5b29f0-9d0f-4ba5-bb1d-a0345e1c6af8 req-e8697f8c-5327-407b-87c5-7dadd90c8ed8 service nova] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] No waiting events found dispatching network-vif-plugged-e316d37b-4368-453f-b8f6-57baa2a3c503 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1142.643139] env[63273]: WARNING nova.compute.manager [req-af5b29f0-9d0f-4ba5-bb1d-a0345e1c6af8 req-e8697f8c-5327-407b-87c5-7dadd90c8ed8 service nova] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Received unexpected event network-vif-plugged-e316d37b-4368-453f-b8f6-57baa2a3c503 for instance with vm_state building and task_state spawning. [ 1142.717399] env[63273]: DEBUG nova.network.neutron [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Successfully updated port: e316d37b-4368-453f-b8f6-57baa2a3c503 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1142.738028] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquiring lock "refresh_cache-fefdf558-7a73-4bae-b57c-b86963189ddb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1142.738028] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquired lock "refresh_cache-fefdf558-7a73-4bae-b57c-b86963189ddb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1142.738028] env[63273]: DEBUG nova.network.neutron [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1142.805844] env[63273]: DEBUG nova.network.neutron [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1142.985785] env[63273]: DEBUG nova.network.neutron [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Updating instance_info_cache with network_info: [{"id": "e316d37b-4368-453f-b8f6-57baa2a3c503", "address": "fa:16:3e:26:6b:c2", "network": {"id": "62c858be-fb3e-4c31-a6fe-ebad7dd303c1", "bridge": "br-int", "label": "tempest-ImagesOneServerTestJSON-514729484-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ad0ab37da3b64969894f8ab378f35ff7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "55520f67-d092-4eb7-940f-d7cceaa1ca1c", "external-id": "nsx-vlan-transportzone-717", "segmentation_id": 717, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape316d37b-43", "ovs_interfaceid": "e316d37b-4368-453f-b8f6-57baa2a3c503", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1143.004077] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Releasing lock "refresh_cache-fefdf558-7a73-4bae-b57c-b86963189ddb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1143.004518] env[63273]: DEBUG nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Instance network_info: |[{"id": "e316d37b-4368-453f-b8f6-57baa2a3c503", "address": "fa:16:3e:26:6b:c2", "network": {"id": "62c858be-fb3e-4c31-a6fe-ebad7dd303c1", "bridge": "br-int", "label": "tempest-ImagesOneServerTestJSON-514729484-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ad0ab37da3b64969894f8ab378f35ff7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "55520f67-d092-4eb7-940f-d7cceaa1ca1c", "external-id": "nsx-vlan-transportzone-717", "segmentation_id": 717, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape316d37b-43", "ovs_interfaceid": "e316d37b-4368-453f-b8f6-57baa2a3c503", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1143.005343] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:26:6b:c2', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '55520f67-d092-4eb7-940f-d7cceaa1ca1c', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'e316d37b-4368-453f-b8f6-57baa2a3c503', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1143.014284] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Creating folder: Project (ad0ab37da3b64969894f8ab378f35ff7). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1143.014284] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-851a743d-1fe6-4d24-9080-6cc6dc33309a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1143.026597] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Created folder: Project (ad0ab37da3b64969894f8ab378f35ff7) in parent group-v986930. [ 1143.026908] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Creating folder: Instances. Parent ref: group-v986992. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1143.027512] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d6b87878-77c1-4c06-b1d9-6f48daf4455c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1143.038653] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Created folder: Instances in parent group-v986992. [ 1143.038933] env[63273]: DEBUG oslo.service.loopingcall [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1143.039161] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1143.039375] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-be0db59b-8b51-4f28-8d5e-8a8946b003d6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1143.061069] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1143.061069] env[63273]: value = "task-5072079" [ 1143.061069] env[63273]: _type = "Task" [ 1143.061069] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1143.070025] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072079, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1143.571838] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072079, 'name': CreateVM_Task, 'duration_secs': 0.292884} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1143.571838] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1143.573334] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1143.573334] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1143.573334] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1143.573334] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-52cd075e-b129-421f-8977-80398405c18d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1143.578956] env[63273]: DEBUG oslo_vmware.api [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Waiting for the task: (returnval){ [ 1143.578956] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52063224-021f-2a8f-e5fd-409a6ffefe77" [ 1143.578956] env[63273]: _type = "Task" [ 1143.578956] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1143.587382] env[63273]: DEBUG oslo_vmware.api [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52063224-021f-2a8f-e5fd-409a6ffefe77, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1144.089141] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1144.089418] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1144.089648] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1144.663219] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "b2b1ee46-307f-40fa-8346-394a39a0a99b" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1144.719599] env[63273]: DEBUG nova.compute.manager [req-add11efd-6d2e-4124-adf8-71d342596b00 req-594f11e8-7269-4783-8bef-80e3b7032027 service nova] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Received event network-changed-e316d37b-4368-453f-b8f6-57baa2a3c503 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1144.719599] env[63273]: DEBUG nova.compute.manager [req-add11efd-6d2e-4124-adf8-71d342596b00 req-594f11e8-7269-4783-8bef-80e3b7032027 service nova] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Refreshing instance network info cache due to event network-changed-e316d37b-4368-453f-b8f6-57baa2a3c503. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1144.719599] env[63273]: DEBUG oslo_concurrency.lockutils [req-add11efd-6d2e-4124-adf8-71d342596b00 req-594f11e8-7269-4783-8bef-80e3b7032027 service nova] Acquiring lock "refresh_cache-fefdf558-7a73-4bae-b57c-b86963189ddb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1144.719743] env[63273]: DEBUG oslo_concurrency.lockutils [req-add11efd-6d2e-4124-adf8-71d342596b00 req-594f11e8-7269-4783-8bef-80e3b7032027 service nova] Acquired lock "refresh_cache-fefdf558-7a73-4bae-b57c-b86963189ddb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1144.719891] env[63273]: DEBUG nova.network.neutron [req-add11efd-6d2e-4124-adf8-71d342596b00 req-594f11e8-7269-4783-8bef-80e3b7032027 service nova] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Refreshing network info cache for port e316d37b-4368-453f-b8f6-57baa2a3c503 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1145.090617] env[63273]: DEBUG nova.network.neutron [req-add11efd-6d2e-4124-adf8-71d342596b00 req-594f11e8-7269-4783-8bef-80e3b7032027 service nova] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Updated VIF entry in instance network info cache for port e316d37b-4368-453f-b8f6-57baa2a3c503. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1145.090999] env[63273]: DEBUG nova.network.neutron [req-add11efd-6d2e-4124-adf8-71d342596b00 req-594f11e8-7269-4783-8bef-80e3b7032027 service nova] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Updating instance_info_cache with network_info: [{"id": "e316d37b-4368-453f-b8f6-57baa2a3c503", "address": "fa:16:3e:26:6b:c2", "network": {"id": "62c858be-fb3e-4c31-a6fe-ebad7dd303c1", "bridge": "br-int", "label": "tempest-ImagesOneServerTestJSON-514729484-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ad0ab37da3b64969894f8ab378f35ff7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "55520f67-d092-4eb7-940f-d7cceaa1ca1c", "external-id": "nsx-vlan-transportzone-717", "segmentation_id": 717, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape316d37b-43", "ovs_interfaceid": "e316d37b-4368-453f-b8f6-57baa2a3c503", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1145.101129] env[63273]: DEBUG oslo_concurrency.lockutils [req-add11efd-6d2e-4124-adf8-71d342596b00 req-594f11e8-7269-4783-8bef-80e3b7032027 service nova] Releasing lock "refresh_cache-fefdf558-7a73-4bae-b57c-b86963189ddb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1153.066381] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1153.066828] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1161.937265] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquiring lock "fefdf558-7a73-4bae-b57c-b86963189ddb" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1162.891777] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1164.176741] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_power_states {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1164.201743] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Getting list of instances from cluster (obj){ [ 1164.201743] env[63273]: value = "domain-c8" [ 1164.201743] env[63273]: _type = "ClusterComputeResource" [ 1164.201743] env[63273]: } {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 1164.203078] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ecc4c70d-31f4-4466-a457-78da13f7544c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1164.220787] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Got total of 10 instances {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 1164.221033] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid f4d86853-25c8-4568-9b3f-8f07bcc5f068 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1164.221208] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 4ac5c733-988f-4428-ad6f-134d9f174e45 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1164.221371] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 164c1d04-3481-4aee-ba56-c80c8ed36e6a {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1164.221524] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid e36789b5-f814-4105-b144-361fef9e0d0e {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1164.221675] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1164.221823] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 13511c60-50cd-44ed-969e-c5fc29b0125e {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1164.221970] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 4a90a188-5167-41ca-8cc0-cce7e65fa5cb {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1164.222141] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1164.222285] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid b2b1ee46-307f-40fa-8346-394a39a0a99b {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1164.222426] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid fefdf558-7a73-4bae-b57c-b86963189ddb {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1164.222789] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1164.223032] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "4ac5c733-988f-4428-ad6f-134d9f174e45" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1164.223247] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1164.223444] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "e36789b5-f814-4105-b144-361fef9e0d0e" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1164.223637] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1164.223835] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "13511c60-50cd-44ed-969e-c5fc29b0125e" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1164.224122] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1164.224354] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1164.224552] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "b2b1ee46-307f-40fa-8346-394a39a0a99b" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1164.224769] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "fefdf558-7a73-4bae-b57c-b86963189ddb" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1164.939856] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1167.891251] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1167.891562] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1167.905761] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1167.905761] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1167.905761] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1167.905761] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1167.906940] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10a06bba-567e-4e6d-a069-b48fdc4af70e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1167.917071] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0468f415-6b00-4fe9-8018-7d82e8e0d983 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1167.932640] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35c83223-fbcd-4fcf-8860-ca7e0688b3f3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1167.940384] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f12dc847-3dd8-41cf-bb96-e8d37cc7f9f9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1167.969283] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180543MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1167.970087] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1167.970087] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1168.155116] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1168.155300] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1168.155427] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1168.155545] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e36789b5-f814-4105-b144-361fef9e0d0e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1168.155662] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1168.155768] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 13511c60-50cd-44ed-969e-c5fc29b0125e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1168.155879] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1168.155991] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1168.156114] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b2b1ee46-307f-40fa-8346-394a39a0a99b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1168.156268] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fefdf558-7a73-4bae-b57c-b86963189ddb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1168.168561] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 083cde6b-b47a-496c-b0e6-4620cba59b9a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.180055] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a31087c9-979d-48dd-975f-322e97f4bbe1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.190588] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 75cfbe32-87f1-49ac-9c6a-4e8838e0971b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.202507] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 99faf6a2-c9a5-43fc-b7b5-9f8c50253278 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.213729] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a081322d-4636-4bfc-90f6-bd0c617a09e8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.226047] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d370386-d8e7-401e-8fc3-c181d74d8125 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.235996] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f65fc3d9-407a-47bd-a038-211b25931d38 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.246802] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 77786a5e-1cb6-4100-a2fc-e67e0140e7ca has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.257500] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d7dbf9b-1ab3-459a-ad02-451f2c522584 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.270493] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.286930] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.297557] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1168.297797] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1168.297965] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '46', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '10', 'num_os_type_None': '10', 'num_proj_70158b9660d14db684be2828e94f8e42': '1', 'io_workload': '10', 'num_proj_f1ef99990442451e927a36cdbb924a86': '1', 'num_proj_8189e2aa9960429596f8a65a91032bed': '1', 'num_proj_bc297a27a74f4303b258579a0b803d81': '1', 'num_proj_6a04825b924a482994b31b7066af5714': '1', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1', 'num_proj_ff52e0a10f1c464f808b6df84185058f': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'num_proj_ad0ab37da3b64969894f8ab378f35ff7': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1168.315332] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Refreshing inventories for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 1168.331359] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Updating ProviderTree inventory for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 1168.331632] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Updating inventory in ProviderTree for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1168.345326] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Refreshing aggregate associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, aggregates: None {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 1168.364518] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Refreshing trait associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NODE,COMPUTE_IMAGE_TYPE_ISO {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 1168.646613] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65775051-1167-4bc3-ad52-fbf4feb971a1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1168.654583] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-82c08d8c-714f-4f65-88ce-a09798f7df72 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1168.693688] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffa7dd87-eba5-4901-9fbe-4855f3df8530 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1168.702797] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-029f7018-87a8-4c0c-a83f-763c83f02da5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1168.718870] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1168.728373] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1168.743056] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1168.743248] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.774s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1169.743988] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1169.744289] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1169.744356] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1169.770998] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1169.771221] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1169.771370] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1169.771498] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1169.771622] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1169.771744] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1169.771863] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1169.771980] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1169.772112] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1169.772231] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1169.772348] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1169.891495] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1169.891725] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1169.891898] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1171.892665] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1173.886242] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1174.969613] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquiring lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1174.970163] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1175.928244] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40bebdcc-6777-4a5a-abef-d5f9b678fc8d tempest-ServersNegativeTestMultiTenantJSON-717800779 tempest-ServersNegativeTestMultiTenantJSON-717800779-project-member] Acquiring lock "e8ed7780-5b17-4693-8cf3-8f75223fce9a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1175.928652] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40bebdcc-6777-4a5a-abef-d5f9b678fc8d tempest-ServersNegativeTestMultiTenantJSON-717800779 tempest-ServersNegativeTestMultiTenantJSON-717800779-project-member] Lock "e8ed7780-5b17-4693-8cf3-8f75223fce9a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1182.107791] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ac9bcd13-2ebb-4f4a-b6a0-989a6c751a86 tempest-ServerTagsTestJSON-853966697 tempest-ServerTagsTestJSON-853966697-project-member] Acquiring lock "7b376dab-bef6-45a7-aead-6d4d0e4a3090" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1182.107791] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ac9bcd13-2ebb-4f4a-b6a0-989a6c751a86 tempest-ServerTagsTestJSON-853966697 tempest-ServerTagsTestJSON-853966697-project-member] Lock "7b376dab-bef6-45a7-aead-6d4d0e4a3090" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1185.217586] env[63273]: WARNING oslo_vmware.rw_handles [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1185.217586] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1185.217586] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1185.217586] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1185.217586] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1185.217586] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1185.217586] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1185.217586] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1185.217586] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1185.217586] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1185.217586] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1185.217586] env[63273]: ERROR oslo_vmware.rw_handles [ 1185.218232] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/7a12cdea-0edd-4e22-a2ea-8cc24f5c03de/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1185.219736] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1185.219977] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Copying Virtual Disk [datastore1] vmware_temp/7a12cdea-0edd-4e22-a2ea-8cc24f5c03de/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/7a12cdea-0edd-4e22-a2ea-8cc24f5c03de/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1185.220291] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-040a0d0b-6842-48df-b6d8-371dffcdafed {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.229120] env[63273]: DEBUG oslo_vmware.api [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Waiting for the task: (returnval){ [ 1185.229120] env[63273]: value = "task-5072080" [ 1185.229120] env[63273]: _type = "Task" [ 1185.229120] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1185.240409] env[63273]: DEBUG oslo_vmware.api [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Task: {'id': task-5072080, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1185.746244] env[63273]: DEBUG oslo_vmware.exceptions [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1185.746244] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1185.746244] env[63273]: ERROR nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1185.746244] env[63273]: Faults: ['InvalidArgument'] [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Traceback (most recent call last): [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] yield resources [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] self.driver.spawn(context, instance, image_meta, [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] self._fetch_image_if_missing(context, vi) [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] image_cache(vi, tmp_image_ds_loc) [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] vm_util.copy_virtual_disk( [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] session._wait_for_task(vmdk_copy_task) [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] return self.wait_for_task(task_ref) [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] return evt.wait() [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] result = hub.switch() [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] return self.greenlet.switch() [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] self.f(*self.args, **self.kw) [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] raise exceptions.translate_fault(task_info.error) [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Faults: ['InvalidArgument'] [ 1185.746244] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] [ 1185.746244] env[63273]: INFO nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Terminating instance [ 1185.747428] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1185.748649] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1185.749685] env[63273]: DEBUG nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1185.749685] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1185.750388] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-37337af3-88cd-4d4a-a9d2-11466f8d78c3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.753378] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfcdc271-c947-440f-a4cc-d88d8af649e9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.763027] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1185.763027] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-78a2910e-e180-4906-b9af-4674fe4375b5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.764842] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1185.765216] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1185.766341] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f8eec512-d7ef-4a4f-af29-053f77f6dcc1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.772758] env[63273]: DEBUG oslo_vmware.api [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Waiting for the task: (returnval){ [ 1185.772758] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]526e228a-3d47-9335-c2d2-26a182c38c97" [ 1185.772758] env[63273]: _type = "Task" [ 1185.772758] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1185.791850] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1185.791850] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Creating directory with path [datastore1] vmware_temp/81bee031-d16a-4d12-abc6-c8f2461d2b77/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1185.791850] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-1b52af1f-82c2-4bf9-8f2d-d69f23936429 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.814240] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Created directory with path [datastore1] vmware_temp/81bee031-d16a-4d12-abc6-c8f2461d2b77/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1185.814240] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Fetch image to [datastore1] vmware_temp/81bee031-d16a-4d12-abc6-c8f2461d2b77/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1185.814240] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/81bee031-d16a-4d12-abc6-c8f2461d2b77/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1185.814240] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8f8318b-4d6a-42a1-9c27-a9c302c6ccb2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.822354] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8870d576-a087-4bc3-b0d4-168a060247c1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.836639] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f2413d5-51b1-4921-bac5-3b6aa8cfea18 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.841045] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1185.841242] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1185.841425] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Deleting the datastore file [datastore1] f4d86853-25c8-4568-9b3f-8f07bcc5f068 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1185.841719] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-0a82d61b-0bb6-4154-a9fc-4f7d50d8b621 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.851352] env[63273]: DEBUG oslo_vmware.api [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Waiting for the task: (returnval){ [ 1185.851352] env[63273]: value = "task-5072082" [ 1185.851352] env[63273]: _type = "Task" [ 1185.851352] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1185.880838] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4386cd8-a371-4c8c-bb87-f32f07e57360 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.890062] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-4b0b1503-0afb-49db-8da5-cca91230da50 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1185.891857] env[63273]: DEBUG oslo_vmware.api [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Task: {'id': task-5072082, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1185.912769] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1186.164351] env[63273]: DEBUG oslo_vmware.rw_handles [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/81bee031-d16a-4d12-abc6-c8f2461d2b77/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1186.228675] env[63273]: DEBUG oslo_vmware.rw_handles [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1186.228949] env[63273]: DEBUG oslo_vmware.rw_handles [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/81bee031-d16a-4d12-abc6-c8f2461d2b77/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1186.387358] env[63273]: DEBUG oslo_vmware.api [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Task: {'id': task-5072082, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.074379} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1186.387742] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1186.387982] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1186.388385] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1186.388652] env[63273]: INFO nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Took 0.64 seconds to destroy the instance on the hypervisor. [ 1186.390904] env[63273]: DEBUG nova.compute.claims [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1186.391236] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1186.391504] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1186.863953] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c23a6db0-0976-4139-a060-2669121996cf {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1186.874234] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09059660-58d5-40ae-8b2f-82f1c9320330 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1186.906018] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b5d4452-dc28-4356-b50a-048fa10b85ba {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1186.914764] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7fd2225d-055a-433b-941e-3d873e46f6f9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1186.928608] env[63273]: DEBUG nova.compute.provider_tree [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1186.938513] env[63273]: DEBUG nova.scheduler.client.report [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1186.953068] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.561s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1186.953608] env[63273]: ERROR nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1186.953608] env[63273]: Faults: ['InvalidArgument'] [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Traceback (most recent call last): [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] self.driver.spawn(context, instance, image_meta, [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] self._fetch_image_if_missing(context, vi) [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] image_cache(vi, tmp_image_ds_loc) [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] vm_util.copy_virtual_disk( [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] session._wait_for_task(vmdk_copy_task) [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] return self.wait_for_task(task_ref) [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] return evt.wait() [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] result = hub.switch() [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] return self.greenlet.switch() [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] self.f(*self.args, **self.kw) [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] raise exceptions.translate_fault(task_info.error) [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Faults: ['InvalidArgument'] [ 1186.953608] env[63273]: ERROR nova.compute.manager [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] [ 1186.954660] env[63273]: DEBUG nova.compute.utils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1186.955759] env[63273]: DEBUG nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Build of instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 was re-scheduled: A specified parameter was not correct: fileType [ 1186.955759] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1186.956141] env[63273]: DEBUG nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1186.956331] env[63273]: DEBUG nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1186.956504] env[63273]: DEBUG nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1186.956670] env[63273]: DEBUG nova.network.neutron [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1187.496979] env[63273]: DEBUG nova.network.neutron [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1187.514679] env[63273]: INFO nova.compute.manager [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Took 0.56 seconds to deallocate network for instance. [ 1187.641613] env[63273]: INFO nova.scheduler.client.report [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Deleted allocations for instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 [ 1187.667539] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9fdd83b6-d172-4786-9e34-62bba7af1c91 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 618.764s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1187.669593] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 421.082s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1187.669593] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Acquiring lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1187.669593] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1187.669894] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1187.673038] env[63273]: INFO nova.compute.manager [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Terminating instance [ 1187.674225] env[63273]: DEBUG nova.compute.manager [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1187.674323] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1187.675439] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f4dfc7a0-1c2e-471b-b780-db038440ee1b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1187.684169] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05c83905-4e13-4469-af39-ea0b38549081 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1187.700011] env[63273]: DEBUG nova.compute.manager [None req-aa0b3fd2-ad90-475a-891b-dbe52dcb4f62 tempest-ServerShowV257Test-1698833534 tempest-ServerShowV257Test-1698833534-project-member] [instance: b8bb414b-3759-41d4-8355-4d34cbad4957] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1187.726824] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f4d86853-25c8-4568-9b3f-8f07bcc5f068 could not be found. [ 1187.726824] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1187.726824] env[63273]: INFO nova.compute.manager [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1187.727069] env[63273]: DEBUG oslo.service.loopingcall [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1187.727757] env[63273]: DEBUG nova.compute.manager [-] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1187.727757] env[63273]: DEBUG nova.network.neutron [-] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1187.734026] env[63273]: DEBUG nova.compute.manager [None req-aa0b3fd2-ad90-475a-891b-dbe52dcb4f62 tempest-ServerShowV257Test-1698833534 tempest-ServerShowV257Test-1698833534-project-member] [instance: b8bb414b-3759-41d4-8355-4d34cbad4957] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1187.757980] env[63273]: DEBUG nova.network.neutron [-] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1187.761347] env[63273]: DEBUG oslo_concurrency.lockutils [None req-aa0b3fd2-ad90-475a-891b-dbe52dcb4f62 tempest-ServerShowV257Test-1698833534 tempest-ServerShowV257Test-1698833534-project-member] Lock "b8bb414b-3759-41d4-8355-4d34cbad4957" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 217.570s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1187.771329] env[63273]: INFO nova.compute.manager [-] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] Took 0.04 seconds to deallocate network for instance. [ 1187.771672] env[63273]: DEBUG nova.compute.manager [None req-e832ae55-ac39-4c48-83d3-96c5a85e2846 tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: 083cde6b-b47a-496c-b0e6-4620cba59b9a] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1187.801553] env[63273]: DEBUG nova.compute.manager [None req-e832ae55-ac39-4c48-83d3-96c5a85e2846 tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: 083cde6b-b47a-496c-b0e6-4620cba59b9a] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1187.829556] env[63273]: DEBUG oslo_concurrency.lockutils [None req-e832ae55-ac39-4c48-83d3-96c5a85e2846 tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "083cde6b-b47a-496c-b0e6-4620cba59b9a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 211.919s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1187.841756] env[63273]: DEBUG nova.compute.manager [None req-97db9751-b513-4160-8343-9072eebed2e2 tempest-ServerGroupTestJSON-323358918 tempest-ServerGroupTestJSON-323358918-project-member] [instance: a31087c9-979d-48dd-975f-322e97f4bbe1] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1187.876375] env[63273]: DEBUG nova.compute.manager [None req-97db9751-b513-4160-8343-9072eebed2e2 tempest-ServerGroupTestJSON-323358918 tempest-ServerGroupTestJSON-323358918-project-member] [instance: a31087c9-979d-48dd-975f-322e97f4bbe1] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1187.889501] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b36e9a89-4949-4274-9720-2e6d38a6c144 tempest-ImagesNegativeTestJSON-782834672 tempest-ImagesNegativeTestJSON-782834672-project-member] Lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.220s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1187.890887] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 23.668s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1187.891831] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f4d86853-25c8-4568-9b3f-8f07bcc5f068] During sync_power_state the instance has a pending task (deleting). Skip. [ 1187.891831] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "f4d86853-25c8-4568-9b3f-8f07bcc5f068" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1187.906194] env[63273]: DEBUG oslo_concurrency.lockutils [None req-97db9751-b513-4160-8343-9072eebed2e2 tempest-ServerGroupTestJSON-323358918 tempest-ServerGroupTestJSON-323358918-project-member] Lock "a31087c9-979d-48dd-975f-322e97f4bbe1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 208.833s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1187.916641] env[63273]: DEBUG nova.compute.manager [None req-c9384bb7-a584-42d6-958b-cd0df3cc5a61 tempest-ServerActionsTestOtherB-1802666129 tempest-ServerActionsTestOtherB-1802666129-project-member] [instance: 75cfbe32-87f1-49ac-9c6a-4e8838e0971b] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1187.942071] env[63273]: DEBUG nova.compute.manager [None req-c9384bb7-a584-42d6-958b-cd0df3cc5a61 tempest-ServerActionsTestOtherB-1802666129 tempest-ServerActionsTestOtherB-1802666129-project-member] [instance: 75cfbe32-87f1-49ac-9c6a-4e8838e0971b] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1187.976095] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c9384bb7-a584-42d6-958b-cd0df3cc5a61 tempest-ServerActionsTestOtherB-1802666129 tempest-ServerActionsTestOtherB-1802666129-project-member] Lock "75cfbe32-87f1-49ac-9c6a-4e8838e0971b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 199.662s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1187.985764] env[63273]: DEBUG nova.compute.manager [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1188.050570] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1188.050570] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1188.052523] env[63273]: INFO nova.compute.claims [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1188.201085] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Acquiring lock "99faf6a2-c9a5-43fc-b7b5-9f8c50253278" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1188.421662] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f36abcb7-ba0a-4a4f-9c22-1a807772b281 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1188.428991] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb55647d-516f-41b6-880b-abb879c7b3f2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1188.463378] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76672a02-431d-4d21-8628-375888b2d7c0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1188.471463] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47655a9a-4418-4bc3-aa5b-b3c67e6a4144 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1188.488491] env[63273]: DEBUG nova.compute.provider_tree [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1188.524406] env[63273]: DEBUG nova.scheduler.client.report [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1188.545417] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.495s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1188.546068] env[63273]: DEBUG nova.compute.manager [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1188.598915] env[63273]: DEBUG nova.compute.claims [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1188.599400] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1188.599400] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1188.971083] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1513f58c-68e3-4adf-90ea-e1f2b77b6108 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1188.981167] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca8e88fd-1a34-4492-ab1e-682e24e521bd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1189.018946] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39b8a841-11ab-4881-8275-0b658cfc3681 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1189.027071] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f65b5777-e5d6-4a17-80cc-d05d708990ce {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1189.043413] env[63273]: DEBUG nova.compute.provider_tree [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1189.059074] env[63273]: DEBUG nova.scheduler.client.report [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1189.074843] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.475s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1189.078092] env[63273]: DEBUG nova.compute.utils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Conflict updating instance 99faf6a2-c9a5-43fc-b7b5-9f8c50253278. Expected: {'task_state': [None]}. Actual: {'task_state': 'deleting'} {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1189.078092] env[63273]: DEBUG nova.compute.manager [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Instance disappeared during build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2505}} [ 1189.078092] env[63273]: DEBUG nova.compute.manager [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1189.078092] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Acquiring lock "refresh_cache-99faf6a2-c9a5-43fc-b7b5-9f8c50253278" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1189.078092] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Acquired lock "refresh_cache-99faf6a2-c9a5-43fc-b7b5-9f8c50253278" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1189.078092] env[63273]: DEBUG nova.network.neutron [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1189.166617] env[63273]: DEBUG nova.network.neutron [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1189.536312] env[63273]: DEBUG nova.network.neutron [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1189.553993] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Releasing lock "refresh_cache-99faf6a2-c9a5-43fc-b7b5-9f8c50253278" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1189.553993] env[63273]: DEBUG nova.compute.manager [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1189.553993] env[63273]: DEBUG nova.compute.manager [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Skipping network deallocation for instance since networking was not requested. {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2277}} [ 1189.669333] env[63273]: INFO nova.scheduler.client.report [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Deleted allocations for instance 99faf6a2-c9a5-43fc-b7b5-9f8c50253278 [ 1189.669710] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a4b10afb-28e3-4339-9b52-6966a1fc29c2 tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Lock "99faf6a2-c9a5-43fc-b7b5-9f8c50253278" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 198.193s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1189.672707] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Lock "99faf6a2-c9a5-43fc-b7b5-9f8c50253278" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 1.472s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1189.674938] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Acquiring lock "99faf6a2-c9a5-43fc-b7b5-9f8c50253278-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1189.675368] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Lock "99faf6a2-c9a5-43fc-b7b5-9f8c50253278-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1189.675586] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Lock "99faf6a2-c9a5-43fc-b7b5-9f8c50253278-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1189.678490] env[63273]: INFO nova.compute.manager [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Terminating instance [ 1189.681480] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Acquiring lock "refresh_cache-99faf6a2-c9a5-43fc-b7b5-9f8c50253278" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1189.683047] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Acquired lock "refresh_cache-99faf6a2-c9a5-43fc-b7b5-9f8c50253278" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1189.683169] env[63273]: DEBUG nova.network.neutron [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1189.713856] env[63273]: DEBUG nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1189.719025] env[63273]: DEBUG nova.network.neutron [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1189.767254] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1189.767524] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1189.769146] env[63273]: INFO nova.compute.claims [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1189.837847] env[63273]: DEBUG nova.network.neutron [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1189.850246] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Releasing lock "refresh_cache-99faf6a2-c9a5-43fc-b7b5-9f8c50253278" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1189.850643] env[63273]: DEBUG nova.compute.manager [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1189.850826] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1189.851365] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8b635857-9300-4ce1-bdf8-1346e037c1b8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1189.864824] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9aa1c41-7770-4612-978c-e3ada482466e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1189.906718] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 99faf6a2-c9a5-43fc-b7b5-9f8c50253278 could not be found. [ 1189.907156] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1189.908485] env[63273]: INFO nova.compute.manager [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Took 0.06 seconds to destroy the instance on the hypervisor. [ 1189.908485] env[63273]: DEBUG oslo.service.loopingcall [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1189.910968] env[63273]: DEBUG nova.compute.manager [-] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1189.911106] env[63273]: DEBUG nova.network.neutron [-] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1189.946513] env[63273]: DEBUG nova.network.neutron [-] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1189.954819] env[63273]: DEBUG nova.network.neutron [-] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1189.967163] env[63273]: INFO nova.compute.manager [-] [instance: 99faf6a2-c9a5-43fc-b7b5-9f8c50253278] Took 0.06 seconds to deallocate network for instance. [ 1190.083026] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d042499-9d59-42f6-a5d2-449f63302d5b tempest-ServerShowV254Test-1967663777 tempest-ServerShowV254Test-1967663777-project-member] Lock "99faf6a2-c9a5-43fc-b7b5-9f8c50253278" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.410s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1190.236989] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11fc77c4-d9b4-4b02-96c5-b302746253dc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1190.247789] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e343d10e-28dc-4498-a25b-e6893bab2b6c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1190.288880] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e89d7512-2c9e-4729-bb2d-0d1a8b05ccec {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1190.297623] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3220404b-e5b5-4356-b734-446e1d7992f1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1190.315064] env[63273]: DEBUG nova.compute.provider_tree [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1190.326840] env[63273]: DEBUG nova.scheduler.client.report [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1190.349018] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.579s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1190.349018] env[63273]: DEBUG nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1190.392705] env[63273]: DEBUG nova.compute.utils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1190.394762] env[63273]: DEBUG nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1190.394762] env[63273]: DEBUG nova.network.neutron [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1190.408236] env[63273]: DEBUG nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1190.505670] env[63273]: DEBUG nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1190.519706] env[63273]: DEBUG nova.policy [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3b7004c3dc964c84a9a5f82c46b853e9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'bc9a82a949b84d2db6b4602db73e0d2e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1190.542196] env[63273]: DEBUG nova.virt.hardware [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1190.542537] env[63273]: DEBUG nova.virt.hardware [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1190.542788] env[63273]: DEBUG nova.virt.hardware [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1190.542910] env[63273]: DEBUG nova.virt.hardware [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1190.543156] env[63273]: DEBUG nova.virt.hardware [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1190.543338] env[63273]: DEBUG nova.virt.hardware [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1190.543567] env[63273]: DEBUG nova.virt.hardware [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1190.543795] env[63273]: DEBUG nova.virt.hardware [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1190.543891] env[63273]: DEBUG nova.virt.hardware [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1190.544063] env[63273]: DEBUG nova.virt.hardware [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1190.544243] env[63273]: DEBUG nova.virt.hardware [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1190.545491] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be473634-a373-45d3-bf8d-985996c022ee {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1190.555430] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3126828d-40d7-4c3d-bede-3d6cc0b461be {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1191.068644] env[63273]: DEBUG nova.network.neutron [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Successfully created port: d2b920b2-575a-428d-bc38-fb1fe48a55b5 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1191.911647] env[63273]: DEBUG nova.compute.manager [req-97853ca4-128d-4c19-93c6-b54d852bf1fc req-698e8b3b-f9eb-4251-ad13-85a10d63daec service nova] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Received event network-vif-plugged-d2b920b2-575a-428d-bc38-fb1fe48a55b5 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1191.911918] env[63273]: DEBUG oslo_concurrency.lockutils [req-97853ca4-128d-4c19-93c6-b54d852bf1fc req-698e8b3b-f9eb-4251-ad13-85a10d63daec service nova] Acquiring lock "a081322d-4636-4bfc-90f6-bd0c617a09e8-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1191.912098] env[63273]: DEBUG oslo_concurrency.lockutils [req-97853ca4-128d-4c19-93c6-b54d852bf1fc req-698e8b3b-f9eb-4251-ad13-85a10d63daec service nova] Lock "a081322d-4636-4bfc-90f6-bd0c617a09e8-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1191.912271] env[63273]: DEBUG oslo_concurrency.lockutils [req-97853ca4-128d-4c19-93c6-b54d852bf1fc req-698e8b3b-f9eb-4251-ad13-85a10d63daec service nova] Lock "a081322d-4636-4bfc-90f6-bd0c617a09e8-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1191.912437] env[63273]: DEBUG nova.compute.manager [req-97853ca4-128d-4c19-93c6-b54d852bf1fc req-698e8b3b-f9eb-4251-ad13-85a10d63daec service nova] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] No waiting events found dispatching network-vif-plugged-d2b920b2-575a-428d-bc38-fb1fe48a55b5 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1191.912603] env[63273]: WARNING nova.compute.manager [req-97853ca4-128d-4c19-93c6-b54d852bf1fc req-698e8b3b-f9eb-4251-ad13-85a10d63daec service nova] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Received unexpected event network-vif-plugged-d2b920b2-575a-428d-bc38-fb1fe48a55b5 for instance with vm_state building and task_state spawning. [ 1192.056010] env[63273]: DEBUG nova.network.neutron [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Successfully updated port: d2b920b2-575a-428d-bc38-fb1fe48a55b5 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1192.075442] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquiring lock "refresh_cache-a081322d-4636-4bfc-90f6-bd0c617a09e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1192.075442] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquired lock "refresh_cache-a081322d-4636-4bfc-90f6-bd0c617a09e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1192.075442] env[63273]: DEBUG nova.network.neutron [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1192.145801] env[63273]: DEBUG nova.network.neutron [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1192.395454] env[63273]: DEBUG nova.network.neutron [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Updating instance_info_cache with network_info: [{"id": "d2b920b2-575a-428d-bc38-fb1fe48a55b5", "address": "fa:16:3e:93:6e:20", "network": {"id": "f85521ff-d467-431f-bde6-1b356862869a", "bridge": "br-int", "label": "tempest-ServersV294TestFqdnHostnames-1813160296-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "bc9a82a949b84d2db6b4602db73e0d2e", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "03ac2c9c-6ad2-4a85-bfab-c7e336df859a", "external-id": "nsx-vlan-transportzone-379", "segmentation_id": 379, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd2b920b2-57", "ovs_interfaceid": "d2b920b2-575a-428d-bc38-fb1fe48a55b5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1192.413381] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Releasing lock "refresh_cache-a081322d-4636-4bfc-90f6-bd0c617a09e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1192.413695] env[63273]: DEBUG nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Instance network_info: |[{"id": "d2b920b2-575a-428d-bc38-fb1fe48a55b5", "address": "fa:16:3e:93:6e:20", "network": {"id": "f85521ff-d467-431f-bde6-1b356862869a", "bridge": "br-int", "label": "tempest-ServersV294TestFqdnHostnames-1813160296-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "bc9a82a949b84d2db6b4602db73e0d2e", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "03ac2c9c-6ad2-4a85-bfab-c7e336df859a", "external-id": "nsx-vlan-transportzone-379", "segmentation_id": 379, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd2b920b2-57", "ovs_interfaceid": "d2b920b2-575a-428d-bc38-fb1fe48a55b5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1192.414127] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:93:6e:20', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '03ac2c9c-6ad2-4a85-bfab-c7e336df859a', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'd2b920b2-575a-428d-bc38-fb1fe48a55b5', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1192.421601] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Creating folder: Project (bc9a82a949b84d2db6b4602db73e0d2e). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1192.422350] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-28d9cd51-b990-47fe-9d99-adcc7e2daf19 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1192.434817] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Created folder: Project (bc9a82a949b84d2db6b4602db73e0d2e) in parent group-v986930. [ 1192.435156] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Creating folder: Instances. Parent ref: group-v986995. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1192.435471] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-71aac0bb-e068-4469-b1a0-dd007fb6c225 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1192.446708] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Created folder: Instances in parent group-v986995. [ 1192.447404] env[63273]: DEBUG oslo.service.loopingcall [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1192.447404] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1192.447710] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-738b712b-b08f-48a9-aab1-40716b603450 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1192.467802] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1192.467802] env[63273]: value = "task-5072085" [ 1192.467802] env[63273]: _type = "Task" [ 1192.467802] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1192.480676] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072085, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1192.977479] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072085, 'name': CreateVM_Task, 'duration_secs': 0.369625} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1192.977899] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1192.985251] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1192.985439] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1192.985806] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1192.986092] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-0eeffff3-f7d6-4bbe-89e4-45974436622c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1192.991652] env[63273]: DEBUG oslo_vmware.api [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Waiting for the task: (returnval){ [ 1192.991652] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e1aebf-6a8e-6cd7-22d2-259e9e336721" [ 1192.991652] env[63273]: _type = "Task" [ 1192.991652] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1193.007685] env[63273]: DEBUG oslo_vmware.api [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e1aebf-6a8e-6cd7-22d2-259e9e336721, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1193.503685] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1193.504048] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1193.504353] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1193.938349] env[63273]: DEBUG nova.compute.manager [req-1332bc85-37eb-4dee-9795-0d7b407c4aa1 req-3e366fcd-5d13-4da5-92a3-e3c42134bbd1 service nova] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Received event network-changed-d2b920b2-575a-428d-bc38-fb1fe48a55b5 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1193.938529] env[63273]: DEBUG nova.compute.manager [req-1332bc85-37eb-4dee-9795-0d7b407c4aa1 req-3e366fcd-5d13-4da5-92a3-e3c42134bbd1 service nova] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Refreshing instance network info cache due to event network-changed-d2b920b2-575a-428d-bc38-fb1fe48a55b5. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1193.938749] env[63273]: DEBUG oslo_concurrency.lockutils [req-1332bc85-37eb-4dee-9795-0d7b407c4aa1 req-3e366fcd-5d13-4da5-92a3-e3c42134bbd1 service nova] Acquiring lock "refresh_cache-a081322d-4636-4bfc-90f6-bd0c617a09e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1193.938879] env[63273]: DEBUG oslo_concurrency.lockutils [req-1332bc85-37eb-4dee-9795-0d7b407c4aa1 req-3e366fcd-5d13-4da5-92a3-e3c42134bbd1 service nova] Acquired lock "refresh_cache-a081322d-4636-4bfc-90f6-bd0c617a09e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1193.939050] env[63273]: DEBUG nova.network.neutron [req-1332bc85-37eb-4dee-9795-0d7b407c4aa1 req-3e366fcd-5d13-4da5-92a3-e3c42134bbd1 service nova] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Refreshing network info cache for port d2b920b2-575a-428d-bc38-fb1fe48a55b5 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1194.466647] env[63273]: DEBUG nova.network.neutron [req-1332bc85-37eb-4dee-9795-0d7b407c4aa1 req-3e366fcd-5d13-4da5-92a3-e3c42134bbd1 service nova] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Updated VIF entry in instance network info cache for port d2b920b2-575a-428d-bc38-fb1fe48a55b5. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1194.467016] env[63273]: DEBUG nova.network.neutron [req-1332bc85-37eb-4dee-9795-0d7b407c4aa1 req-3e366fcd-5d13-4da5-92a3-e3c42134bbd1 service nova] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Updating instance_info_cache with network_info: [{"id": "d2b920b2-575a-428d-bc38-fb1fe48a55b5", "address": "fa:16:3e:93:6e:20", "network": {"id": "f85521ff-d467-431f-bde6-1b356862869a", "bridge": "br-int", "label": "tempest-ServersV294TestFqdnHostnames-1813160296-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "bc9a82a949b84d2db6b4602db73e0d2e", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "03ac2c9c-6ad2-4a85-bfab-c7e336df859a", "external-id": "nsx-vlan-transportzone-379", "segmentation_id": 379, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd2b920b2-57", "ovs_interfaceid": "d2b920b2-575a-428d-bc38-fb1fe48a55b5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1194.477808] env[63273]: DEBUG oslo_concurrency.lockutils [req-1332bc85-37eb-4dee-9795-0d7b407c4aa1 req-3e366fcd-5d13-4da5-92a3-e3c42134bbd1 service nova] Releasing lock "refresh_cache-a081322d-4636-4bfc-90f6-bd0c617a09e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1197.374696] env[63273]: DEBUG oslo_concurrency.lockutils [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquiring lock "a081322d-4636-4bfc-90f6-bd0c617a09e8" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1206.310990] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "fa31b240-8bad-48ba-8339-155dc6acb265" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1206.311505] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fa31b240-8bad-48ba-8339-155dc6acb265" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1207.578791] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d6af161-8417-4ed0-b070-527beae04be5 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquiring lock "7dd30edb-74fc-47c8-8506-40bd1470f0f5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1207.579441] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d6af161-8417-4ed0-b070-527beae04be5 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "7dd30edb-74fc-47c8-8506-40bd1470f0f5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1207.652031] env[63273]: DEBUG oslo_concurrency.lockutils [None req-14f74b91-389d-42c3-b65d-481a81582242 tempest-ServerAddressesTestJSON-647465110 tempest-ServerAddressesTestJSON-647465110-project-member] Acquiring lock "2e353c90-12e0-49dc-aa45-a916f65d6c97" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1207.652031] env[63273]: DEBUG oslo_concurrency.lockutils [None req-14f74b91-389d-42c3-b65d-481a81582242 tempest-ServerAddressesTestJSON-647465110 tempest-ServerAddressesTestJSON-647465110-project-member] Lock "2e353c90-12e0-49dc-aa45-a916f65d6c97" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1211.007059] env[63273]: DEBUG oslo_concurrency.lockutils [None req-df99a5f9-d1d8-4d31-b624-5d4c41d544b4 tempest-ServerActionsV293TestJSON-1119087857 tempest-ServerActionsV293TestJSON-1119087857-project-member] Acquiring lock "6b83e614-0581-4d5b-8cc9-e08daa0b738a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1211.007382] env[63273]: DEBUG oslo_concurrency.lockutils [None req-df99a5f9-d1d8-4d31-b624-5d4c41d544b4 tempest-ServerActionsV293TestJSON-1119087857 tempest-ServerActionsV293TestJSON-1119087857-project-member] Lock "6b83e614-0581-4d5b-8cc9-e08daa0b738a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1217.420226] env[63273]: DEBUG oslo_concurrency.lockutils [None req-db777207-2a5e-471b-9ca3-154a7b533edb tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "92a8d50b-adce-475e-a6f1-da3645afc20a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1217.420721] env[63273]: DEBUG oslo_concurrency.lockutils [None req-db777207-2a5e-471b-9ca3-154a7b533edb tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "92a8d50b-adce-475e-a6f1-da3645afc20a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1222.750922] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1a505c25-bccd-40ad-998b-7947d490a9ab tempest-AttachInterfacesUnderV243Test-1148727018 tempest-AttachInterfacesUnderV243Test-1148727018-project-member] Acquiring lock "185173ef-30f6-4ddb-bcae-c6e4436d131e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1222.751259] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1a505c25-bccd-40ad-998b-7947d490a9ab tempest-AttachInterfacesUnderV243Test-1148727018 tempest-AttachInterfacesUnderV243Test-1148727018-project-member] Lock "185173ef-30f6-4ddb-bcae-c6e4436d131e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1223.891336] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1225.891369] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1227.891886] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1227.892313] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1227.904219] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1227.904446] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1227.904617] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1227.904779] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1227.908580] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-034823cf-eba5-4da5-8a5f-8b8d340e1edd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1227.915702] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fee6641d-24c1-4300-bc3a-9223a05446c1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1227.929582] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3db50444-a115-4bd6-ac0a-37b6647f0b22 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1227.935798] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-837390b8-5527-4568-bd07-0d3f71f04844 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1227.965642] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180490MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1227.965818] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1227.966030] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1228.043784] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1228.043956] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1228.044086] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e36789b5-f814-4105-b144-361fef9e0d0e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1228.044218] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1228.044331] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 13511c60-50cd-44ed-969e-c5fc29b0125e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1228.044450] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1228.044595] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1228.044689] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b2b1ee46-307f-40fa-8346-394a39a0a99b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1228.044834] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fefdf558-7a73-4bae-b57c-b86963189ddb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1228.044958] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a081322d-4636-4bfc-90f6-bd0c617a09e8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1228.058024] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.068498] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.080157] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.090386] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.101428] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e8ed7780-5b17-4693-8cf3-8f75223fce9a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.111671] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7b376dab-bef6-45a7-aead-6d4d0e4a3090 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.121768] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fa31b240-8bad-48ba-8339-155dc6acb265 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.131379] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7dd30edb-74fc-47c8-8506-40bd1470f0f5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.141470] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2e353c90-12e0-49dc-aa45-a916f65d6c97 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.151300] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 6b83e614-0581-4d5b-8cc9-e08daa0b738a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.161776] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92a8d50b-adce-475e-a6f1-da3645afc20a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.172687] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 185173ef-30f6-4ddb-bcae-c6e4436d131e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1228.172930] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1228.173112] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '52', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '10', 'num_os_type_None': '10', 'num_proj_f1ef99990442451e927a36cdbb924a86': '1', 'io_workload': '10', 'num_proj_8189e2aa9960429596f8a65a91032bed': '1', 'num_proj_bc297a27a74f4303b258579a0b803d81': '1', 'num_proj_6a04825b924a482994b31b7066af5714': '1', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1', 'num_proj_ff52e0a10f1c464f808b6df84185058f': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'num_proj_ad0ab37da3b64969894f8ab378f35ff7': '1', 'num_proj_bc9a82a949b84d2db6b4602db73e0d2e': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1228.449263] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4fa3f20d-5f6d-4fdf-9d07-4afa6f04d8a3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1228.457351] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7cd584f-9381-4b19-ab31-d5e5358a6890 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1228.493299] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ef0d07a-be8a-4fad-b61d-d40e6b3b519c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1228.501604] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36db08ce-1dc2-49de-b60c-cf46f106a7c3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1228.515191] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1228.523817] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1228.541452] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1228.541697] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.576s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1230.541239] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1230.887296] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1230.911046] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1230.911182] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1230.911237] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1230.932736] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1230.932911] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1230.933052] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1230.933178] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1230.933298] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1230.933417] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1230.933533] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1230.933651] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1230.933770] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1230.933883] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1230.934007] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1230.934854] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1230.935058] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1232.893039] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1233.053828] env[63273]: WARNING oslo_vmware.rw_handles [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1233.053828] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1233.053828] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1233.053828] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1233.053828] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1233.053828] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1233.053828] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1233.053828] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1233.053828] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1233.053828] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1233.053828] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1233.053828] env[63273]: ERROR oslo_vmware.rw_handles [ 1233.054357] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/81bee031-d16a-4d12-abc6-c8f2461d2b77/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1233.056089] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1233.056336] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Copying Virtual Disk [datastore1] vmware_temp/81bee031-d16a-4d12-abc6-c8f2461d2b77/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/81bee031-d16a-4d12-abc6-c8f2461d2b77/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1233.056617] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-f789f93e-4854-4689-9756-d8348019d773 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1233.064742] env[63273]: DEBUG oslo_vmware.api [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Waiting for the task: (returnval){ [ 1233.064742] env[63273]: value = "task-5072096" [ 1233.064742] env[63273]: _type = "Task" [ 1233.064742] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1233.073180] env[63273]: DEBUG oslo_vmware.api [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Task: {'id': task-5072096, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1233.575272] env[63273]: DEBUG oslo_vmware.exceptions [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1233.575565] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1233.576253] env[63273]: ERROR nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1233.576253] env[63273]: Faults: ['InvalidArgument'] [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Traceback (most recent call last): [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] yield resources [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] self.driver.spawn(context, instance, image_meta, [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] self._fetch_image_if_missing(context, vi) [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] image_cache(vi, tmp_image_ds_loc) [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] vm_util.copy_virtual_disk( [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] session._wait_for_task(vmdk_copy_task) [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] return self.wait_for_task(task_ref) [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] return evt.wait() [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] result = hub.switch() [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] return self.greenlet.switch() [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] self.f(*self.args, **self.kw) [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] raise exceptions.translate_fault(task_info.error) [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Faults: ['InvalidArgument'] [ 1233.576253] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] [ 1233.577538] env[63273]: INFO nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Terminating instance [ 1233.578244] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1233.578447] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1233.578687] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-561d0900-9676-4596-979f-69589f026ef1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1233.581163] env[63273]: DEBUG nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1233.581352] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1233.582095] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e7fcacd-efa1-402d-b528-3d20d3dd9c37 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1233.589553] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1233.589847] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-c1e85338-2c46-4495-a8f2-3d7dc208064d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1233.592235] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1233.592411] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1233.593427] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-25c22dbc-c12b-46bc-b42c-c1ba3b0a1ae9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1233.600329] env[63273]: DEBUG oslo_vmware.api [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Waiting for the task: (returnval){ [ 1233.600329] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52c86b9c-c2fd-8f97-6049-a2b7d37a5d91" [ 1233.600329] env[63273]: _type = "Task" [ 1233.600329] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1233.608419] env[63273]: DEBUG oslo_vmware.api [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52c86b9c-c2fd-8f97-6049-a2b7d37a5d91, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1233.662394] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1233.662640] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1233.662852] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Deleting the datastore file [datastore1] 4ac5c733-988f-4428-ad6f-134d9f174e45 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1233.663312] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-515c4710-4b14-4bf7-94cc-6cc7ba5bb8f1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1233.670595] env[63273]: DEBUG oslo_vmware.api [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Waiting for the task: (returnval){ [ 1233.670595] env[63273]: value = "task-5072098" [ 1233.670595] env[63273]: _type = "Task" [ 1233.670595] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1233.679912] env[63273]: DEBUG oslo_vmware.api [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Task: {'id': task-5072098, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1234.110849] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1234.111235] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Creating directory with path [datastore1] vmware_temp/588379b7-6eee-4ce8-84a0-f2163b059841/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1234.111441] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-afe8ccb9-a584-41e3-b44b-81fbf92d74fb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1234.123233] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Created directory with path [datastore1] vmware_temp/588379b7-6eee-4ce8-84a0-f2163b059841/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1234.123436] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Fetch image to [datastore1] vmware_temp/588379b7-6eee-4ce8-84a0-f2163b059841/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1234.123605] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/588379b7-6eee-4ce8-84a0-f2163b059841/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1234.124417] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f954853-48d5-42f1-80fa-3e07d6dac1e2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1234.131996] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a87973e3-fa2a-4ead-aef3-78e48a3eedc2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1234.141660] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fde51ece-e558-44c7-94b6-1c158d6860f7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1234.175074] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c36605f7-a131-4d55-aa08-180fbb14a62d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1234.188444] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-0209ce61-02b4-4738-86db-1852463fd150 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1234.190340] env[63273]: DEBUG oslo_vmware.api [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Task: {'id': task-5072098, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.083113} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1234.190586] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1234.190767] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1234.190937] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1234.191125] env[63273]: INFO nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Took 0.61 seconds to destroy the instance on the hypervisor. [ 1234.193727] env[63273]: DEBUG nova.compute.claims [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1234.193915] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1234.194146] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1234.219028] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1234.386787] env[63273]: DEBUG oslo_vmware.rw_handles [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/588379b7-6eee-4ce8-84a0-f2163b059841/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1234.445597] env[63273]: DEBUG oslo_vmware.rw_handles [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1234.445822] env[63273]: DEBUG oslo_vmware.rw_handles [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/588379b7-6eee-4ce8-84a0-f2163b059841/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1234.595024] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79f1f1b4-8ced-4bb8-8849-ec12b68c2953 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1234.603411] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-722d002f-b57a-4a24-a8dd-b99c90f48a80 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1234.633414] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccfca123-764d-4340-a179-f59c27a52afb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1234.641900] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3cef2e38-3029-49a2-9c72-776448617c8f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1234.655605] env[63273]: DEBUG nova.compute.provider_tree [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1234.666499] env[63273]: DEBUG nova.scheduler.client.report [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1234.682520] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.488s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1234.683374] env[63273]: ERROR nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1234.683374] env[63273]: Faults: ['InvalidArgument'] [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Traceback (most recent call last): [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] self.driver.spawn(context, instance, image_meta, [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] self._fetch_image_if_missing(context, vi) [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] image_cache(vi, tmp_image_ds_loc) [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] vm_util.copy_virtual_disk( [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] session._wait_for_task(vmdk_copy_task) [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] return self.wait_for_task(task_ref) [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] return evt.wait() [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] result = hub.switch() [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] return self.greenlet.switch() [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] self.f(*self.args, **self.kw) [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] raise exceptions.translate_fault(task_info.error) [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Faults: ['InvalidArgument'] [ 1234.683374] env[63273]: ERROR nova.compute.manager [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] [ 1234.689024] env[63273]: DEBUG nova.compute.utils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1234.689024] env[63273]: DEBUG nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Build of instance 4ac5c733-988f-4428-ad6f-134d9f174e45 was re-scheduled: A specified parameter was not correct: fileType [ 1234.689024] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1234.689024] env[63273]: DEBUG nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1234.689024] env[63273]: DEBUG nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1234.689024] env[63273]: DEBUG nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1234.689024] env[63273]: DEBUG nova.network.neutron [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1235.123275] env[63273]: DEBUG nova.network.neutron [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1235.136578] env[63273]: INFO nova.compute.manager [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Took 0.45 seconds to deallocate network for instance. [ 1235.267737] env[63273]: INFO nova.scheduler.client.report [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Deleted allocations for instance 4ac5c733-988f-4428-ad6f-134d9f174e45 [ 1235.292564] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c7008e71-1aca-4582-99a5-f4c0601bfb80 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Lock "4ac5c733-988f-4428-ad6f-134d9f174e45" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 665.356s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1235.293954] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Lock "4ac5c733-988f-4428-ad6f-134d9f174e45" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 468.397s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1235.294323] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Acquiring lock "4ac5c733-988f-4428-ad6f-134d9f174e45-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1235.294622] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Lock "4ac5c733-988f-4428-ad6f-134d9f174e45-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1235.294718] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Lock "4ac5c733-988f-4428-ad6f-134d9f174e45-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1235.299343] env[63273]: INFO nova.compute.manager [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Terminating instance [ 1235.302299] env[63273]: DEBUG nova.compute.manager [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1235.302559] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1235.302924] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c07c61fb-29e6-4120-8096-49f2106497be {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1235.314139] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-774b0124-7880-4b3b-a759-b4b7237031d5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1235.328930] env[63273]: DEBUG nova.compute.manager [None req-6ef1ec96-65a8-4255-842d-368764dc3f04 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 5d370386-d8e7-401e-8fc3-c181d74d8125] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1235.352501] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4ac5c733-988f-4428-ad6f-134d9f174e45 could not be found. [ 1235.352873] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1235.352949] env[63273]: INFO nova.compute.manager [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1235.353165] env[63273]: DEBUG oslo.service.loopingcall [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1235.353449] env[63273]: DEBUG nova.compute.manager [-] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1235.353545] env[63273]: DEBUG nova.network.neutron [-] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1235.361648] env[63273]: DEBUG nova.compute.manager [None req-6ef1ec96-65a8-4255-842d-368764dc3f04 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 5d370386-d8e7-401e-8fc3-c181d74d8125] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1235.393254] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6ef1ec96-65a8-4255-842d-368764dc3f04 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "5d370386-d8e7-401e-8fc3-c181d74d8125" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 228.113s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1235.396132] env[63273]: DEBUG nova.network.neutron [-] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1235.403713] env[63273]: DEBUG nova.compute.manager [None req-f6d49f4a-4f3e-4553-b1aa-a53e13e531a7 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: f65fc3d9-407a-47bd-a038-211b25931d38] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1235.406806] env[63273]: INFO nova.compute.manager [-] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] Took 0.05 seconds to deallocate network for instance. [ 1235.442756] env[63273]: DEBUG nova.compute.manager [None req-f6d49f4a-4f3e-4553-b1aa-a53e13e531a7 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: f65fc3d9-407a-47bd-a038-211b25931d38] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1235.470863] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f6d49f4a-4f3e-4553-b1aa-a53e13e531a7 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "f65fc3d9-407a-47bd-a038-211b25931d38" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 224.710s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1235.482654] env[63273]: DEBUG nova.compute.manager [None req-b0c8c1fe-cc6d-442f-bde4-9fff100444d6 tempest-AttachVolumeTestJSON-293439156 tempest-AttachVolumeTestJSON-293439156-project-member] [instance: 77786a5e-1cb6-4100-a2fc-e67e0140e7ca] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1235.519604] env[63273]: DEBUG nova.compute.manager [None req-b0c8c1fe-cc6d-442f-bde4-9fff100444d6 tempest-AttachVolumeTestJSON-293439156 tempest-AttachVolumeTestJSON-293439156-project-member] [instance: 77786a5e-1cb6-4100-a2fc-e67e0140e7ca] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1235.540057] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b53c46ca-eac7-4b5e-a92c-df18f91ce270 tempest-ServersTestJSON-1419549153 tempest-ServersTestJSON-1419549153-project-member] Lock "4ac5c733-988f-4428-ad6f-134d9f174e45" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.246s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1235.541475] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "4ac5c733-988f-4428-ad6f-134d9f174e45" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 71.318s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1235.541650] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4ac5c733-988f-4428-ad6f-134d9f174e45] During sync_power_state the instance has a pending task (deleting). Skip. [ 1235.541877] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "4ac5c733-988f-4428-ad6f-134d9f174e45" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1235.548827] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b0c8c1fe-cc6d-442f-bde4-9fff100444d6 tempest-AttachVolumeTestJSON-293439156 tempest-AttachVolumeTestJSON-293439156-project-member] Lock "77786a5e-1cb6-4100-a2fc-e67e0140e7ca" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 224.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1235.560197] env[63273]: DEBUG nova.compute.manager [None req-a792310a-0dc8-49f4-b3f4-9d8aba6bc69e tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] [instance: 5d7dbf9b-1ab3-459a-ad02-451f2c522584] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1235.587247] env[63273]: DEBUG nova.compute.manager [None req-a792310a-0dc8-49f4-b3f4-9d8aba6bc69e tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] [instance: 5d7dbf9b-1ab3-459a-ad02-451f2c522584] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1235.614802] env[63273]: DEBUG oslo_concurrency.lockutils [None req-a792310a-0dc8-49f4-b3f4-9d8aba6bc69e tempest-AttachInterfacesTestJSON-790861651 tempest-AttachInterfacesTestJSON-790861651-project-member] Lock "5d7dbf9b-1ab3-459a-ad02-451f2c522584" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 218.515s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1235.628709] env[63273]: DEBUG nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1235.693240] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1235.693506] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1235.695067] env[63273]: INFO nova.compute.claims [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1235.887181] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1236.065226] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab8d1537-39f2-46da-a4c7-ba673fa2a09c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.073800] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac73e1aa-15c3-491c-9e72-d7e55dcf5158 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.107042] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-559df74d-7f0f-43d9-98b7-a11a817a386d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.115361] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47418978-65a7-4f57-a9b9-34d5b4888bb7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.131045] env[63273]: DEBUG nova.compute.provider_tree [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1236.141236] env[63273]: DEBUG nova.scheduler.client.report [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1236.159985] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.466s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1236.160553] env[63273]: DEBUG nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1236.206894] env[63273]: DEBUG nova.compute.utils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1236.209163] env[63273]: DEBUG nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1236.209431] env[63273]: DEBUG nova.network.neutron [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1236.220283] env[63273]: DEBUG nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1236.274210] env[63273]: DEBUG nova.policy [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f41e3b4457e6405f86406048d62c3e61', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '71b2d87fd7ff416db331d00f7375e59b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1236.292127] env[63273]: DEBUG nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1236.321184] env[63273]: DEBUG nova.virt.hardware [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1236.321445] env[63273]: DEBUG nova.virt.hardware [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1236.321603] env[63273]: DEBUG nova.virt.hardware [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1236.321782] env[63273]: DEBUG nova.virt.hardware [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1236.321932] env[63273]: DEBUG nova.virt.hardware [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1236.322092] env[63273]: DEBUG nova.virt.hardware [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1236.322302] env[63273]: DEBUG nova.virt.hardware [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1236.322461] env[63273]: DEBUG nova.virt.hardware [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1236.322624] env[63273]: DEBUG nova.virt.hardware [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1236.322786] env[63273]: DEBUG nova.virt.hardware [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1236.322956] env[63273]: DEBUG nova.virt.hardware [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1236.323836] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f54d1a10-f9ca-429e-9ebf-9c7a67ab117e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.333654] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-493341e5-8a04-4c2d-b642-a9afe21d1d4c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.744453] env[63273]: DEBUG nova.network.neutron [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Successfully created port: e6255b79-78e6-45bd-bdfb-0a957be9f5ee {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1237.631560] env[63273]: DEBUG nova.compute.manager [req-3ffd43dd-a9d1-4614-98b1-30470bb7d269 req-0348b3cd-de60-4b37-bd14-cf0bc9bf1899 service nova] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Received event network-vif-plugged-e6255b79-78e6-45bd-bdfb-0a957be9f5ee {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1237.631853] env[63273]: DEBUG oslo_concurrency.lockutils [req-3ffd43dd-a9d1-4614-98b1-30470bb7d269 req-0348b3cd-de60-4b37-bd14-cf0bc9bf1899 service nova] Acquiring lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1237.631994] env[63273]: DEBUG oslo_concurrency.lockutils [req-3ffd43dd-a9d1-4614-98b1-30470bb7d269 req-0348b3cd-de60-4b37-bd14-cf0bc9bf1899 service nova] Lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1237.632178] env[63273]: DEBUG oslo_concurrency.lockutils [req-3ffd43dd-a9d1-4614-98b1-30470bb7d269 req-0348b3cd-de60-4b37-bd14-cf0bc9bf1899 service nova] Lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1237.632341] env[63273]: DEBUG nova.compute.manager [req-3ffd43dd-a9d1-4614-98b1-30470bb7d269 req-0348b3cd-de60-4b37-bd14-cf0bc9bf1899 service nova] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] No waiting events found dispatching network-vif-plugged-e6255b79-78e6-45bd-bdfb-0a957be9f5ee {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1237.632501] env[63273]: WARNING nova.compute.manager [req-3ffd43dd-a9d1-4614-98b1-30470bb7d269 req-0348b3cd-de60-4b37-bd14-cf0bc9bf1899 service nova] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Received unexpected event network-vif-plugged-e6255b79-78e6-45bd-bdfb-0a957be9f5ee for instance with vm_state building and task_state spawning. [ 1237.715931] env[63273]: DEBUG nova.network.neutron [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Successfully updated port: e6255b79-78e6-45bd-bdfb-0a957be9f5ee {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1237.727586] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "refresh_cache-f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1237.727788] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquired lock "refresh_cache-f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1237.727946] env[63273]: DEBUG nova.network.neutron [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1237.788191] env[63273]: DEBUG nova.network.neutron [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1237.987182] env[63273]: DEBUG nova.network.neutron [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Updating instance_info_cache with network_info: [{"id": "e6255b79-78e6-45bd-bdfb-0a957be9f5ee", "address": "fa:16:3e:7f:ec:6e", "network": {"id": "c237f546-9020-408f-b5ff-275fe78c70f7", "bridge": "br-int", "label": "tempest-ServersAdminTestJSON-381996572-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "71b2d87fd7ff416db331d00f7375e59b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "9a1e09ef-7c9c-45d9-9bf4-55b913524948", "external-id": "nsx-vlan-transportzone-466", "segmentation_id": 466, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape6255b79-78", "ovs_interfaceid": "e6255b79-78e6-45bd-bdfb-0a957be9f5ee", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1238.005186] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Releasing lock "refresh_cache-f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1238.005524] env[63273]: DEBUG nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Instance network_info: |[{"id": "e6255b79-78e6-45bd-bdfb-0a957be9f5ee", "address": "fa:16:3e:7f:ec:6e", "network": {"id": "c237f546-9020-408f-b5ff-275fe78c70f7", "bridge": "br-int", "label": "tempest-ServersAdminTestJSON-381996572-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "71b2d87fd7ff416db331d00f7375e59b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "9a1e09ef-7c9c-45d9-9bf4-55b913524948", "external-id": "nsx-vlan-transportzone-466", "segmentation_id": 466, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape6255b79-78", "ovs_interfaceid": "e6255b79-78e6-45bd-bdfb-0a957be9f5ee", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1238.006013] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:7f:ec:6e', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '9a1e09ef-7c9c-45d9-9bf4-55b913524948', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'e6255b79-78e6-45bd-bdfb-0a957be9f5ee', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1238.013605] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Creating folder: Project (71b2d87fd7ff416db331d00f7375e59b). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1238.014208] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d580913d-09c2-4b1d-a09b-de5467cbe848 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1238.025856] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Created folder: Project (71b2d87fd7ff416db331d00f7375e59b) in parent group-v986930. [ 1238.026338] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Creating folder: Instances. Parent ref: group-v987002. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1238.026490] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-52e5942d-15f1-4595-89f5-6a203968596e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1238.035261] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Created folder: Instances in parent group-v987002. [ 1238.035419] env[63273]: DEBUG oslo.service.loopingcall [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1238.035619] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1238.035839] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-31091513-43bc-4272-b111-ba9cc3c91c22 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1238.057037] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1238.057037] env[63273]: value = "task-5072101" [ 1238.057037] env[63273]: _type = "Task" [ 1238.057037] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1238.064014] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072101, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1238.565981] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072101, 'name': CreateVM_Task, 'duration_secs': 0.297169} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1238.566225] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1238.566906] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1238.567082] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1238.567433] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1238.567731] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-87369852-174f-4a29-b015-4e7c119da467 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1238.572891] env[63273]: DEBUG oslo_vmware.api [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for the task: (returnval){ [ 1238.572891] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52d4a1b6-9b11-9e61-4b05-1bb59368eff5" [ 1238.572891] env[63273]: _type = "Task" [ 1238.572891] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1238.581110] env[63273]: DEBUG oslo_vmware.api [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52d4a1b6-9b11-9e61-4b05-1bb59368eff5, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1239.084016] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1239.084310] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1239.084542] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1239.660717] env[63273]: DEBUG nova.compute.manager [req-ca7d4509-c58b-4b80-9031-f89f0cc3c85c req-aee0b332-37af-4f51-859a-09f0e52b13b9 service nova] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Received event network-changed-e6255b79-78e6-45bd-bdfb-0a957be9f5ee {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1239.660819] env[63273]: DEBUG nova.compute.manager [req-ca7d4509-c58b-4b80-9031-f89f0cc3c85c req-aee0b332-37af-4f51-859a-09f0e52b13b9 service nova] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Refreshing instance network info cache due to event network-changed-e6255b79-78e6-45bd-bdfb-0a957be9f5ee. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1239.661025] env[63273]: DEBUG oslo_concurrency.lockutils [req-ca7d4509-c58b-4b80-9031-f89f0cc3c85c req-aee0b332-37af-4f51-859a-09f0e52b13b9 service nova] Acquiring lock "refresh_cache-f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1239.661179] env[63273]: DEBUG oslo_concurrency.lockutils [req-ca7d4509-c58b-4b80-9031-f89f0cc3c85c req-aee0b332-37af-4f51-859a-09f0e52b13b9 service nova] Acquired lock "refresh_cache-f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1239.661340] env[63273]: DEBUG nova.network.neutron [req-ca7d4509-c58b-4b80-9031-f89f0cc3c85c req-aee0b332-37af-4f51-859a-09f0e52b13b9 service nova] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Refreshing network info cache for port e6255b79-78e6-45bd-bdfb-0a957be9f5ee {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1239.986907] env[63273]: DEBUG nova.network.neutron [req-ca7d4509-c58b-4b80-9031-f89f0cc3c85c req-aee0b332-37af-4f51-859a-09f0e52b13b9 service nova] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Updated VIF entry in instance network info cache for port e6255b79-78e6-45bd-bdfb-0a957be9f5ee. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1239.987296] env[63273]: DEBUG nova.network.neutron [req-ca7d4509-c58b-4b80-9031-f89f0cc3c85c req-aee0b332-37af-4f51-859a-09f0e52b13b9 service nova] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Updating instance_info_cache with network_info: [{"id": "e6255b79-78e6-45bd-bdfb-0a957be9f5ee", "address": "fa:16:3e:7f:ec:6e", "network": {"id": "c237f546-9020-408f-b5ff-275fe78c70f7", "bridge": "br-int", "label": "tempest-ServersAdminTestJSON-381996572-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "71b2d87fd7ff416db331d00f7375e59b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "9a1e09ef-7c9c-45d9-9bf4-55b913524948", "external-id": "nsx-vlan-transportzone-466", "segmentation_id": 466, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape6255b79-78", "ovs_interfaceid": "e6255b79-78e6-45bd-bdfb-0a957be9f5ee", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1239.997620] env[63273]: DEBUG oslo_concurrency.lockutils [req-ca7d4509-c58b-4b80-9031-f89f0cc3c85c req-aee0b332-37af-4f51-859a-09f0e52b13b9 service nova] Releasing lock "refresh_cache-f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1283.070698] env[63273]: WARNING oslo_vmware.rw_handles [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1283.070698] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1283.070698] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1283.070698] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1283.070698] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1283.070698] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1283.070698] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1283.070698] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1283.070698] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1283.070698] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1283.070698] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1283.070698] env[63273]: ERROR oslo_vmware.rw_handles [ 1283.071371] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/588379b7-6eee-4ce8-84a0-f2163b059841/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1283.073421] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1283.073718] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Copying Virtual Disk [datastore1] vmware_temp/588379b7-6eee-4ce8-84a0-f2163b059841/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/588379b7-6eee-4ce8-84a0-f2163b059841/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1283.074029] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-eb511609-87c8-4487-afee-8fcf1c034ecb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.082325] env[63273]: DEBUG oslo_vmware.api [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Waiting for the task: (returnval){ [ 1283.082325] env[63273]: value = "task-5072102" [ 1283.082325] env[63273]: _type = "Task" [ 1283.082325] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1283.091707] env[63273]: DEBUG oslo_vmware.api [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Task: {'id': task-5072102, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1283.592938] env[63273]: DEBUG oslo_vmware.exceptions [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1283.593255] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1283.593824] env[63273]: ERROR nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1283.593824] env[63273]: Faults: ['InvalidArgument'] [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Traceback (most recent call last): [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] yield resources [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] self.driver.spawn(context, instance, image_meta, [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] self._fetch_image_if_missing(context, vi) [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] image_cache(vi, tmp_image_ds_loc) [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] vm_util.copy_virtual_disk( [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] session._wait_for_task(vmdk_copy_task) [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] return self.wait_for_task(task_ref) [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] return evt.wait() [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] result = hub.switch() [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] return self.greenlet.switch() [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] self.f(*self.args, **self.kw) [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] raise exceptions.translate_fault(task_info.error) [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Faults: ['InvalidArgument'] [ 1283.593824] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] [ 1283.594780] env[63273]: INFO nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Terminating instance [ 1283.595732] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1283.595938] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1283.596215] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-bf7be2bb-3bfd-431b-866e-9d7e3f5a282f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.599486] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1283.599767] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1283.600416] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c863db65-aa6b-4598-845d-8507247f1d6f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.607974] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1283.608306] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-7c1419fe-970f-458d-984c-caa7af16e790 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.610815] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1283.610983] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1283.611993] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-4405d17f-315c-4799-81ba-5ff85b0ee950 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.617199] env[63273]: DEBUG oslo_vmware.api [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Waiting for the task: (returnval){ [ 1283.617199] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]525f0b8c-143e-7cf3-36fc-77f847cf83ae" [ 1283.617199] env[63273]: _type = "Task" [ 1283.617199] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1283.625285] env[63273]: DEBUG oslo_vmware.api [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]525f0b8c-143e-7cf3-36fc-77f847cf83ae, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1283.686740] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1283.686960] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1283.687163] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Deleting the datastore file [datastore1] 164c1d04-3481-4aee-ba56-c80c8ed36e6a {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1283.687500] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-05b8a568-c0f2-4a09-8567-198330d15892 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.695160] env[63273]: DEBUG oslo_vmware.api [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Waiting for the task: (returnval){ [ 1283.695160] env[63273]: value = "task-5072104" [ 1283.695160] env[63273]: _type = "Task" [ 1283.695160] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1283.704078] env[63273]: DEBUG oslo_vmware.api [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Task: {'id': task-5072104, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1284.132051] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1284.132051] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Creating directory with path [datastore1] vmware_temp/f40e32a8-0915-449a-b5af-5f98cec0bc5a/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1284.132051] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-65c7314e-25bf-4abc-a1b0-befafef2fded {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.142452] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Created directory with path [datastore1] vmware_temp/f40e32a8-0915-449a-b5af-5f98cec0bc5a/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1284.142667] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Fetch image to [datastore1] vmware_temp/f40e32a8-0915-449a-b5af-5f98cec0bc5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1284.142833] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/f40e32a8-0915-449a-b5af-5f98cec0bc5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1284.143634] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0890079-e588-485b-bb59-0e6b04b57480 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.150776] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4132a120-d2cf-498f-b3c2-644e011bad70 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.162761] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d37427d-ef46-4ed3-88b8-a2c67082156b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.194504] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a2696c8-e0ee-4b0a-9ade-fe1bd9dca941 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.206400] env[63273]: DEBUG oslo_vmware.api [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Task: {'id': task-5072104, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.084247} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1284.206940] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1284.207139] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1284.207314] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1284.207489] env[63273]: INFO nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Took 0.61 seconds to destroy the instance on the hypervisor. [ 1284.209110] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-cd20030e-be29-42c2-be86-0d3da7290498 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.211935] env[63273]: DEBUG nova.compute.claims [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1284.212176] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1284.212350] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1284.232029] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1284.351783] env[63273]: DEBUG oslo_vmware.rw_handles [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f40e32a8-0915-449a-b5af-5f98cec0bc5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1284.411988] env[63273]: DEBUG oslo_vmware.rw_handles [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1284.412207] env[63273]: DEBUG oslo_vmware.rw_handles [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f40e32a8-0915-449a-b5af-5f98cec0bc5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1284.574013] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e5bae36-97cf-44ff-a6a6-c796fda90ec9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.582319] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ec7f2da-de86-4b22-877a-63e474d9dc3d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.614041] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd9c8026-e8ba-4ad8-83e2-1f97c479f919 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.621955] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-576b5477-882d-4a83-853d-17803a4b9db9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.635614] env[63273]: DEBUG nova.compute.provider_tree [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1284.645591] env[63273]: DEBUG nova.scheduler.client.report [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1284.664346] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.452s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1284.664969] env[63273]: ERROR nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1284.664969] env[63273]: Faults: ['InvalidArgument'] [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Traceback (most recent call last): [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] self.driver.spawn(context, instance, image_meta, [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] self._fetch_image_if_missing(context, vi) [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] image_cache(vi, tmp_image_ds_loc) [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] vm_util.copy_virtual_disk( [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] session._wait_for_task(vmdk_copy_task) [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] return self.wait_for_task(task_ref) [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] return evt.wait() [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] result = hub.switch() [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] return self.greenlet.switch() [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] self.f(*self.args, **self.kw) [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] raise exceptions.translate_fault(task_info.error) [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Faults: ['InvalidArgument'] [ 1284.664969] env[63273]: ERROR nova.compute.manager [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] [ 1284.666253] env[63273]: DEBUG nova.compute.utils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1284.667248] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Build of instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a was re-scheduled: A specified parameter was not correct: fileType [ 1284.667248] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1284.667617] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1284.667787] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1284.667957] env[63273]: DEBUG nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1284.668136] env[63273]: DEBUG nova.network.neutron [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1285.112501] env[63273]: DEBUG nova.network.neutron [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1285.128900] env[63273]: INFO nova.compute.manager [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Took 0.46 seconds to deallocate network for instance. [ 1285.227512] env[63273]: INFO nova.scheduler.client.report [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Deleted allocations for instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a [ 1285.261688] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a596ed5-2f0c-4199-a10c-c3734323cf95 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 676.196s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1285.262946] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 476.686s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1285.263197] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Acquiring lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1285.263409] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1285.263575] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1285.266817] env[63273]: INFO nova.compute.manager [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Terminating instance [ 1285.269901] env[63273]: DEBUG nova.compute.manager [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1285.270871] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1285.270871] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-48f9e7fc-8d21-4d4b-970e-402eaf05a2cc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1285.280453] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-115472f9-22cd-403d-8157-adf9e92c694b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1285.293793] env[63273]: DEBUG nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1285.314021] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 164c1d04-3481-4aee-ba56-c80c8ed36e6a could not be found. [ 1285.314021] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1285.314021] env[63273]: INFO nova.compute.manager [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1285.314021] env[63273]: DEBUG oslo.service.loopingcall [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1285.314021] env[63273]: DEBUG nova.compute.manager [-] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1285.314021] env[63273]: DEBUG nova.network.neutron [-] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1285.343175] env[63273]: DEBUG nova.network.neutron [-] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1285.351840] env[63273]: INFO nova.compute.manager [-] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] Took 0.04 seconds to deallocate network for instance. [ 1285.370649] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1285.370910] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1285.373308] env[63273]: INFO nova.compute.claims [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1285.459774] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7e4fa02c-6dab-426b-8997-438f77705c22 tempest-ListServersNegativeTestJSON-254879234 tempest-ListServersNegativeTestJSON-254879234-project-member] Lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.196s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1285.461073] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 121.237s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1285.461073] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 164c1d04-3481-4aee-ba56-c80c8ed36e6a] During sync_power_state the instance has a pending task (deleting). Skip. [ 1285.461213] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "164c1d04-3481-4aee-ba56-c80c8ed36e6a" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1285.694638] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-775e04c3-6cab-4673-80b8-b070a695c495 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1285.702846] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8917d3c-77e4-4e04-a50c-1a1d646b49f0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1285.733994] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-188f52b5-ad8f-4bb8-b193-067cff6b9c72 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1285.741495] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb243ecb-61d8-4e8e-84e2-1f05e20da604 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1285.755483] env[63273]: DEBUG nova.compute.provider_tree [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1285.765279] env[63273]: DEBUG nova.scheduler.client.report [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1285.780511] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.409s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1285.780875] env[63273]: DEBUG nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1285.817649] env[63273]: DEBUG nova.compute.utils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1285.818997] env[63273]: DEBUG nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1285.819600] env[63273]: DEBUG nova.network.neutron [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1285.828920] env[63273]: DEBUG nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1285.888594] env[63273]: DEBUG nova.policy [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f41e3b4457e6405f86406048d62c3e61', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '71b2d87fd7ff416db331d00f7375e59b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1285.890665] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1285.899984] env[63273]: DEBUG nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1285.925987] env[63273]: DEBUG nova.virt.hardware [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1285.925987] env[63273]: DEBUG nova.virt.hardware [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1285.925987] env[63273]: DEBUG nova.virt.hardware [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1285.926235] env[63273]: DEBUG nova.virt.hardware [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1285.926235] env[63273]: DEBUG nova.virt.hardware [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1285.926335] env[63273]: DEBUG nova.virt.hardware [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1285.926540] env[63273]: DEBUG nova.virt.hardware [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1285.926699] env[63273]: DEBUG nova.virt.hardware [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1285.926865] env[63273]: DEBUG nova.virt.hardware [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1285.927051] env[63273]: DEBUG nova.virt.hardware [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1285.927215] env[63273]: DEBUG nova.virt.hardware [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1285.928100] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33561ee1-c38c-40c8-8074-e2120c74e53d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1285.936730] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-845e3417-7600-44bd-aa02-a4d7da19659e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1286.263988] env[63273]: DEBUG nova.network.neutron [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Successfully created port: c9e7a489-f3a2-4716-aad7-ded93e5b35e5 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1287.069767] env[63273]: DEBUG nova.network.neutron [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Successfully updated port: c9e7a489-f3a2-4716-aad7-ded93e5b35e5 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1287.084535] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "refresh_cache-5d5955e6-416d-4eaa-b23e-c886ba4d4f26" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1287.084687] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquired lock "refresh_cache-5d5955e6-416d-4eaa-b23e-c886ba4d4f26" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1287.084838] env[63273]: DEBUG nova.network.neutron [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1287.129034] env[63273]: DEBUG nova.network.neutron [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1287.155993] env[63273]: DEBUG nova.compute.manager [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Received event network-vif-plugged-c9e7a489-f3a2-4716-aad7-ded93e5b35e5 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1287.156367] env[63273]: DEBUG oslo_concurrency.lockutils [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] Acquiring lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1287.156709] env[63273]: DEBUG oslo_concurrency.lockutils [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] Lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1287.157015] env[63273]: DEBUG oslo_concurrency.lockutils [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] Lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1287.157314] env[63273]: DEBUG nova.compute.manager [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] No waiting events found dispatching network-vif-plugged-c9e7a489-f3a2-4716-aad7-ded93e5b35e5 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1287.157638] env[63273]: WARNING nova.compute.manager [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Received unexpected event network-vif-plugged-c9e7a489-f3a2-4716-aad7-ded93e5b35e5 for instance with vm_state building and task_state spawning. [ 1287.157888] env[63273]: DEBUG nova.compute.manager [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Received event network-changed-c9e7a489-f3a2-4716-aad7-ded93e5b35e5 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1287.158174] env[63273]: DEBUG nova.compute.manager [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Refreshing instance network info cache due to event network-changed-c9e7a489-f3a2-4716-aad7-ded93e5b35e5. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1287.158467] env[63273]: DEBUG oslo_concurrency.lockutils [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] Acquiring lock "refresh_cache-5d5955e6-416d-4eaa-b23e-c886ba4d4f26" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1287.332643] env[63273]: DEBUG nova.network.neutron [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Updating instance_info_cache with network_info: [{"id": "c9e7a489-f3a2-4716-aad7-ded93e5b35e5", "address": "fa:16:3e:f3:bd:49", "network": {"id": "c237f546-9020-408f-b5ff-275fe78c70f7", "bridge": "br-int", "label": "tempest-ServersAdminTestJSON-381996572-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "71b2d87fd7ff416db331d00f7375e59b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "9a1e09ef-7c9c-45d9-9bf4-55b913524948", "external-id": "nsx-vlan-transportzone-466", "segmentation_id": 466, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc9e7a489-f3", "ovs_interfaceid": "c9e7a489-f3a2-4716-aad7-ded93e5b35e5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1287.347457] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Releasing lock "refresh_cache-5d5955e6-416d-4eaa-b23e-c886ba4d4f26" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1287.347773] env[63273]: DEBUG nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Instance network_info: |[{"id": "c9e7a489-f3a2-4716-aad7-ded93e5b35e5", "address": "fa:16:3e:f3:bd:49", "network": {"id": "c237f546-9020-408f-b5ff-275fe78c70f7", "bridge": "br-int", "label": "tempest-ServersAdminTestJSON-381996572-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "71b2d87fd7ff416db331d00f7375e59b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "9a1e09ef-7c9c-45d9-9bf4-55b913524948", "external-id": "nsx-vlan-transportzone-466", "segmentation_id": 466, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc9e7a489-f3", "ovs_interfaceid": "c9e7a489-f3a2-4716-aad7-ded93e5b35e5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1287.348103] env[63273]: DEBUG oslo_concurrency.lockutils [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] Acquired lock "refresh_cache-5d5955e6-416d-4eaa-b23e-c886ba4d4f26" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1287.348346] env[63273]: DEBUG nova.network.neutron [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Refreshing network info cache for port c9e7a489-f3a2-4716-aad7-ded93e5b35e5 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1287.349642] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:f3:bd:49', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '9a1e09ef-7c9c-45d9-9bf4-55b913524948', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'c9e7a489-f3a2-4716-aad7-ded93e5b35e5', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1287.358367] env[63273]: DEBUG oslo.service.loopingcall [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1287.359668] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1287.362307] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-60707ffa-133b-4f0a-bf44-b7a400ddbe2c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1287.385615] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1287.385615] env[63273]: value = "task-5072105" [ 1287.385615] env[63273]: _type = "Task" [ 1287.385615] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1287.395384] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072105, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1287.721183] env[63273]: DEBUG nova.network.neutron [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Updated VIF entry in instance network info cache for port c9e7a489-f3a2-4716-aad7-ded93e5b35e5. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1287.721558] env[63273]: DEBUG nova.network.neutron [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Updating instance_info_cache with network_info: [{"id": "c9e7a489-f3a2-4716-aad7-ded93e5b35e5", "address": "fa:16:3e:f3:bd:49", "network": {"id": "c237f546-9020-408f-b5ff-275fe78c70f7", "bridge": "br-int", "label": "tempest-ServersAdminTestJSON-381996572-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "71b2d87fd7ff416db331d00f7375e59b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "9a1e09ef-7c9c-45d9-9bf4-55b913524948", "external-id": "nsx-vlan-transportzone-466", "segmentation_id": 466, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc9e7a489-f3", "ovs_interfaceid": "c9e7a489-f3a2-4716-aad7-ded93e5b35e5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1287.733805] env[63273]: DEBUG oslo_concurrency.lockutils [req-c018f0e0-1ea5-4320-8bf2-17afc6368441 req-13281150-e9bc-45b5-a9ff-4e6f70d35346 service nova] Releasing lock "refresh_cache-5d5955e6-416d-4eaa-b23e-c886ba4d4f26" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1287.891038] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1287.891152] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1287.891373] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1287.895647] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072105, 'name': CreateVM_Task} progress is 25%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1287.904515] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1287.904727] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1287.904896] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1287.905064] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1287.906157] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-247ad668-5a45-4423-aea5-2c72cc9e7ff4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1287.914821] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c126774-b619-4163-b8ab-e6a8139e5958 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1287.929314] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-072a1c46-c103-45b1-94cb-336c43ccd21a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1287.936340] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b792da95-a30e-420f-8f5a-b9f2ae154039 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1287.967375] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180545MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1287.967525] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1287.967732] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1288.044597] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e36789b5-f814-4105-b144-361fef9e0d0e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1288.044597] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1288.044597] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 13511c60-50cd-44ed-969e-c5fc29b0125e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1288.044597] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1288.044792] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1288.044792] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b2b1ee46-307f-40fa-8346-394a39a0a99b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1288.044859] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fefdf558-7a73-4bae-b57c-b86963189ddb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1288.044954] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a081322d-4636-4bfc-90f6-bd0c617a09e8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1288.045093] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1288.045223] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1288.057374] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1288.068853] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1288.079110] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e8ed7780-5b17-4693-8cf3-8f75223fce9a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1288.090363] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7b376dab-bef6-45a7-aead-6d4d0e4a3090 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1288.102536] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fa31b240-8bad-48ba-8339-155dc6acb265 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1288.114131] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7dd30edb-74fc-47c8-8506-40bd1470f0f5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1288.129518] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2e353c90-12e0-49dc-aa45-a916f65d6c97 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1288.139607] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 6b83e614-0581-4d5b-8cc9-e08daa0b738a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1288.150826] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92a8d50b-adce-475e-a6f1-da3645afc20a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1288.162697] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 185173ef-30f6-4ddb-bcae-c6e4436d131e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1288.162942] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1288.163115] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '58', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '8', 'num_os_type_None': '10', 'num_proj_bc297a27a74f4303b258579a0b803d81': '1', 'io_workload': '10', 'num_proj_6a04825b924a482994b31b7066af5714': '1', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1', 'num_proj_ff52e0a10f1c464f808b6df84185058f': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'num_proj_ad0ab37da3b64969894f8ab378f35ff7': '1', 'num_proj_bc9a82a949b84d2db6b4602db73e0d2e': '1', 'num_task_spawning': '2', 'num_proj_71b2d87fd7ff416db331d00f7375e59b': '2'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1288.396444] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072105, 'name': CreateVM_Task} progress is 25%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1288.422897] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9659efa7-e55f-4646-b8d7-8bf6e5e4237d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1288.430872] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c24c9284-7286-4fcc-9c49-0cd7e0c869b6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1288.460912] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81221c42-762f-49ce-92cc-41949b0f2204 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1288.468995] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a7165cd-9632-4ba7-b314-67006411cdc3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1288.484098] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1288.495917] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1288.512265] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1288.512462] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.545s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1288.897256] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072105, 'name': CreateVM_Task} progress is 25%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1289.398727] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072105, 'name': CreateVM_Task} progress is 25%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1289.899818] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072105, 'name': CreateVM_Task} progress is 25%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1290.402328] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072105, 'name': CreateVM_Task} progress is 25%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1290.513257] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1290.901855] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072105, 'name': CreateVM_Task} progress is 25%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1291.402124] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072105, 'name': CreateVM_Task} progress is 25%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1291.892239] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1291.892541] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1291.892541] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1291.903628] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072105, 'name': CreateVM_Task} progress is 25%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1291.916603] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1291.916929] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1291.917037] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1291.917146] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1291.917306] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1291.917458] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1291.917608] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1291.918012] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1291.918012] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1291.918012] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1291.918257] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1291.918868] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1291.919293] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1292.263743] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "1da174da-b4e6-437c-a538-53fc78b4282f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1292.264523] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "1da174da-b4e6-437c-a538-53fc78b4282f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1292.403314] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072105, 'name': CreateVM_Task, 'duration_secs': 4.82993} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1292.403499] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1292.404183] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1292.404350] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1292.404682] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1292.404937] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5d672850-b2f0-48ba-a142-2dcc537e2fd3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1292.409508] env[63273]: DEBUG oslo_vmware.api [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for the task: (returnval){ [ 1292.409508] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52f4c924-c242-d1b4-0462-f18781d105f3" [ 1292.409508] env[63273]: _type = "Task" [ 1292.409508] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1292.417437] env[63273]: DEBUG oslo_vmware.api [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52f4c924-c242-d1b4-0462-f18781d105f3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1292.920491] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1292.920782] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1292.920975] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1294.891668] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1296.887132] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1302.694569] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1330.262778] env[63273]: WARNING oslo_vmware.rw_handles [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1330.262778] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1330.262778] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1330.262778] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1330.262778] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1330.262778] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1330.262778] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1330.262778] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1330.262778] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1330.262778] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1330.262778] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1330.262778] env[63273]: ERROR oslo_vmware.rw_handles [ 1330.263451] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/f40e32a8-0915-449a-b5af-5f98cec0bc5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1330.265731] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1330.265976] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Copying Virtual Disk [datastore1] vmware_temp/f40e32a8-0915-449a-b5af-5f98cec0bc5a/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/f40e32a8-0915-449a-b5af-5f98cec0bc5a/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1330.266308] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-10a057fb-1a61-47d8-a0a3-dca461f2e204 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1330.274677] env[63273]: DEBUG oslo_vmware.api [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Waiting for the task: (returnval){ [ 1330.274677] env[63273]: value = "task-5072106" [ 1330.274677] env[63273]: _type = "Task" [ 1330.274677] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1330.283461] env[63273]: DEBUG oslo_vmware.api [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Task: {'id': task-5072106, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1330.787064] env[63273]: DEBUG oslo_vmware.exceptions [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1330.787217] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1330.787783] env[63273]: ERROR nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1330.787783] env[63273]: Faults: ['InvalidArgument'] [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Traceback (most recent call last): [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] yield resources [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] self.driver.spawn(context, instance, image_meta, [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] self._fetch_image_if_missing(context, vi) [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] image_cache(vi, tmp_image_ds_loc) [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] vm_util.copy_virtual_disk( [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] session._wait_for_task(vmdk_copy_task) [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] return self.wait_for_task(task_ref) [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] return evt.wait() [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] result = hub.switch() [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] return self.greenlet.switch() [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] self.f(*self.args, **self.kw) [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] raise exceptions.translate_fault(task_info.error) [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Faults: ['InvalidArgument'] [ 1330.787783] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] [ 1330.788781] env[63273]: INFO nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Terminating instance [ 1330.789776] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1330.789985] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1330.791076] env[63273]: DEBUG nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1330.791265] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1330.791497] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a9b0a0ae-119b-460a-8de3-2995335a4ffb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1330.793892] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3648024-f8f2-4f58-be06-77de4efa36a4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1330.800992] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1330.801275] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-cb84bfde-58bb-4129-94e9-505613714c3e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1330.803490] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1330.803667] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1330.804669] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-074f072f-5b90-49a9-ab23-1bf2a928d650 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1330.810601] env[63273]: DEBUG oslo_vmware.api [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Waiting for the task: (returnval){ [ 1330.810601] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5238f822-54f6-cc34-3359-6d3db343616b" [ 1330.810601] env[63273]: _type = "Task" [ 1330.810601] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1330.819014] env[63273]: DEBUG oslo_vmware.api [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5238f822-54f6-cc34-3359-6d3db343616b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1330.883425] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1330.883647] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1330.883830] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Deleting the datastore file [datastore1] e36789b5-f814-4105-b144-361fef9e0d0e {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1330.884180] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-bf8eaa36-68bf-40e2-b2a6-b6f0274245d8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1330.891207] env[63273]: DEBUG oslo_vmware.api [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Waiting for the task: (returnval){ [ 1330.891207] env[63273]: value = "task-5072108" [ 1330.891207] env[63273]: _type = "Task" [ 1330.891207] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1330.899804] env[63273]: DEBUG oslo_vmware.api [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Task: {'id': task-5072108, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1331.322192] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1331.322487] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Creating directory with path [datastore1] vmware_temp/769e31ab-b327-465a-9813-e013ed8b426e/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1331.322720] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-b10b43fb-7a70-447f-9efe-a10509e47401 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.335069] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Created directory with path [datastore1] vmware_temp/769e31ab-b327-465a-9813-e013ed8b426e/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1331.335269] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Fetch image to [datastore1] vmware_temp/769e31ab-b327-465a-9813-e013ed8b426e/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1331.335436] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/769e31ab-b327-465a-9813-e013ed8b426e/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1331.336282] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45a6ff50-67e8-46d8-9741-c64f96013d68 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.343690] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4bed3bcb-7d1a-4553-b933-2b39defeb8c8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.353077] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1a5cfd4-f635-4fc4-a16a-6bd3be0b00b9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.385391] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e88a754-e506-40ab-80b0-18deb171007f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.394846] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-48bcd8ca-8e40-429c-a52d-e6552e507b11 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.401544] env[63273]: DEBUG oslo_vmware.api [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Task: {'id': task-5072108, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.069064} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1331.401833] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1331.402037] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1331.402216] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1331.402388] env[63273]: INFO nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Took 0.61 seconds to destroy the instance on the hypervisor. [ 1331.404983] env[63273]: DEBUG nova.compute.claims [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1331.404983] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1331.406025] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1331.420928] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1331.570129] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1331.571791] env[63273]: ERROR nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Instance failed to spawn: nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Traceback (most recent call last): [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] result = getattr(controller, method)(*args, **kwargs) [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._get(image_id) [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] resp, body = self.http_client.get(url, headers=header) [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.request(url, 'GET', **kwargs) [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._handle_response(resp) [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise exc.from_response(resp, resp.content) [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] During handling of the above exception, another exception occurred: [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Traceback (most recent call last): [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] yield resources [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self.driver.spawn(context, instance, image_meta, [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._fetch_image_if_missing(context, vi) [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] image_fetch(context, vi, tmp_image_ds_loc) [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] images.fetch_image( [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1331.571791] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] metadata = IMAGE_API.get(context, image_ref) [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return session.show(context, image_id, [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] _reraise_translated_image_exception(image_id) [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise new_exc.with_traceback(exc_trace) [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] result = getattr(controller, method)(*args, **kwargs) [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._get(image_id) [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] resp, body = self.http_client.get(url, headers=header) [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.request(url, 'GET', **kwargs) [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._handle_response(resp) [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise exc.from_response(resp, resp.content) [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1331.572820] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1331.572820] env[63273]: INFO nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Terminating instance [ 1331.573764] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1331.573977] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1331.574618] env[63273]: DEBUG nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1331.574806] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1331.575053] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-9b1096b5-60c2-4990-a138-41b0332c6537 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.577764] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a874d031-1894-4075-ae76-04f409b52e26 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.587938] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1331.588539] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-a3367406-070e-469d-9810-dd8bcc4ed00f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.591074] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1331.591267] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1331.592261] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8a1460ce-b836-4be7-ba67-b46580c6c419 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.601686] env[63273]: DEBUG oslo_vmware.api [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Waiting for the task: (returnval){ [ 1331.601686] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52b4ede7-ac36-948c-68f8-3ffea448812b" [ 1331.601686] env[63273]: _type = "Task" [ 1331.601686] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1331.610320] env[63273]: DEBUG oslo_vmware.api [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52b4ede7-ac36-948c-68f8-3ffea448812b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1331.657880] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1331.658137] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1331.658304] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Deleting the datastore file [datastore1] d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1331.658622] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-bda6b190-e5da-41c0-a59c-9092f14d40ba {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.669641] env[63273]: DEBUG oslo_vmware.api [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Waiting for the task: (returnval){ [ 1331.669641] env[63273]: value = "task-5072110" [ 1331.669641] env[63273]: _type = "Task" [ 1331.669641] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1331.677829] env[63273]: DEBUG oslo_vmware.api [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Task: {'id': task-5072110, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1331.763111] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ae1ad8b-24b3-44d5-8c2f-b947dc3daddd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.770702] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1fcfb9bc-4d38-4869-a8fe-165b5097aeb7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.802612] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29d52903-12fa-4c50-a2f9-dc053c6241cb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.810503] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a32c487-ee40-43c7-b7d5-ce01c771418e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1331.824944] env[63273]: DEBUG nova.compute.provider_tree [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1331.834889] env[63273]: DEBUG nova.scheduler.client.report [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1331.849498] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.444s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1331.850103] env[63273]: ERROR nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1331.850103] env[63273]: Faults: ['InvalidArgument'] [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Traceback (most recent call last): [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] self.driver.spawn(context, instance, image_meta, [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] self._fetch_image_if_missing(context, vi) [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] image_cache(vi, tmp_image_ds_loc) [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] vm_util.copy_virtual_disk( [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] session._wait_for_task(vmdk_copy_task) [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] return self.wait_for_task(task_ref) [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] return evt.wait() [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] result = hub.switch() [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] return self.greenlet.switch() [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] self.f(*self.args, **self.kw) [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] raise exceptions.translate_fault(task_info.error) [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Faults: ['InvalidArgument'] [ 1331.850103] env[63273]: ERROR nova.compute.manager [instance: e36789b5-f814-4105-b144-361fef9e0d0e] [ 1331.850844] env[63273]: DEBUG nova.compute.utils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1331.852693] env[63273]: DEBUG nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Build of instance e36789b5-f814-4105-b144-361fef9e0d0e was re-scheduled: A specified parameter was not correct: fileType [ 1331.852693] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1331.853087] env[63273]: DEBUG nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1331.853271] env[63273]: DEBUG nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1331.853441] env[63273]: DEBUG nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1331.853604] env[63273]: DEBUG nova.network.neutron [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1332.112850] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1332.113294] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Creating directory with path [datastore1] vmware_temp/d658030f-7971-4232-af01-d16443cc1268/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1332.113653] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a21d78a6-8c7a-45e6-820a-6fd2a8c73c95 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.126436] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Created directory with path [datastore1] vmware_temp/d658030f-7971-4232-af01-d16443cc1268/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1332.126850] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Fetch image to [datastore1] vmware_temp/d658030f-7971-4232-af01-d16443cc1268/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1332.127224] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/d658030f-7971-4232-af01-d16443cc1268/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1332.128139] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3015214-10b6-482e-80e5-c584bc67b5f2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.135770] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f6da5b1-db8d-4819-8129-adc9167bb26f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.145822] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b8f12c9-b661-40c8-a46d-0d83cebfa2a9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.184017] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1526e339-bb0a-478e-9315-21a97d8e575f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.192684] env[63273]: DEBUG oslo_vmware.api [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Task: {'id': task-5072110, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.070627} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1332.193386] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1332.193710] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1332.193988] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1332.194298] env[63273]: INFO nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1332.196088] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-74e8b174-3c7a-464b-9ef9-d20cc51f358a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.198246] env[63273]: DEBUG nova.compute.claims [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1332.198538] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1332.198870] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1332.224173] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1332.270486] env[63273]: DEBUG nova.network.neutron [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1332.286738] env[63273]: INFO nova.compute.manager [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Took 0.43 seconds to deallocate network for instance. [ 1332.298959] env[63273]: DEBUG oslo_vmware.rw_handles [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d658030f-7971-4232-af01-d16443cc1268/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1332.364100] env[63273]: DEBUG oslo_vmware.rw_handles [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1332.364383] env[63273]: DEBUG oslo_vmware.rw_handles [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d658030f-7971-4232-af01-d16443cc1268/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1332.424700] env[63273]: INFO nova.scheduler.client.report [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Deleted allocations for instance e36789b5-f814-4105-b144-361fef9e0d0e [ 1332.453619] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9b7762aa-8bf8-4e7a-9e5b-6f8b73004eb7 tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Lock "e36789b5-f814-4105-b144-361fef9e0d0e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 653.758s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1332.455900] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Lock "e36789b5-f814-4105-b144-361fef9e0d0e" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 457.352s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1332.456202] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Acquiring lock "e36789b5-f814-4105-b144-361fef9e0d0e-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1332.456392] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Lock "e36789b5-f814-4105-b144-361fef9e0d0e-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1332.456563] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Lock "e36789b5-f814-4105-b144-361fef9e0d0e-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1332.461986] env[63273]: INFO nova.compute.manager [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Terminating instance [ 1332.464512] env[63273]: DEBUG nova.compute.manager [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1332.464738] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1332.465891] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-69c8b0ad-928d-453d-8093-5c471cf84cdf {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.477148] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de4e9acd-7502-4f1b-a9e9-998ec04220de {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.494577] env[63273]: DEBUG nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1332.518727] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e36789b5-f814-4105-b144-361fef9e0d0e could not be found. [ 1332.518815] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1332.519135] env[63273]: INFO nova.compute.manager [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1332.519332] env[63273]: DEBUG oslo.service.loopingcall [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1332.521961] env[63273]: DEBUG nova.compute.manager [-] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1332.522070] env[63273]: DEBUG nova.network.neutron [-] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1332.555395] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1332.558545] env[63273]: DEBUG nova.network.neutron [-] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1332.571634] env[63273]: INFO nova.compute.manager [-] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] Took 0.05 seconds to deallocate network for instance. [ 1332.654391] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef039274-305d-43e3-aabf-13d4eb05ab08 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.663414] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf39f7e0-9019-4df7-a023-977858eb4278 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.694713] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6bc114a5-fa80-4cdd-9206-cece2fceef1f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.700420] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d7e38ad-79ac-41e5-8a70-8a4fbc8b492d tempest-ServersTestManualDisk-434081010 tempest-ServersTestManualDisk-434081010-project-member] Lock "e36789b5-f814-4105-b144-361fef9e0d0e" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.244s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1332.701310] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "e36789b5-f814-4105-b144-361fef9e0d0e" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 168.478s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1332.701392] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: e36789b5-f814-4105-b144-361fef9e0d0e] During sync_power_state the instance has a pending task (deleting). Skip. [ 1332.701563] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "e36789b5-f814-4105-b144-361fef9e0d0e" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1332.705179] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1867e276-0205-4a50-a457-3cd03824f60f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.719907] env[63273]: DEBUG nova.compute.provider_tree [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1332.728414] env[63273]: DEBUG nova.scheduler.client.report [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1332.743731] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.545s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1332.744449] env[63273]: ERROR nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Failed to build and run instance: nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Traceback (most recent call last): [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] result = getattr(controller, method)(*args, **kwargs) [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._get(image_id) [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] resp, body = self.http_client.get(url, headers=header) [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.request(url, 'GET', **kwargs) [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._handle_response(resp) [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise exc.from_response(resp, resp.content) [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] During handling of the above exception, another exception occurred: [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Traceback (most recent call last): [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self.driver.spawn(context, instance, image_meta, [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._fetch_image_if_missing(context, vi) [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] image_fetch(context, vi, tmp_image_ds_loc) [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] images.fetch_image( [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] metadata = IMAGE_API.get(context, image_ref) [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1332.744449] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return session.show(context, image_id, [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] _reraise_translated_image_exception(image_id) [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise new_exc.with_traceback(exc_trace) [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] result = getattr(controller, method)(*args, **kwargs) [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._get(image_id) [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] resp, body = self.http_client.get(url, headers=header) [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.request(url, 'GET', **kwargs) [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._handle_response(resp) [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise exc.from_response(resp, resp.content) [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1332.745369] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.745369] env[63273]: DEBUG nova.compute.utils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1332.746400] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.192s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1332.747899] env[63273]: INFO nova.compute.claims [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1332.751245] env[63273]: DEBUG nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Build of instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 was re-scheduled: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1332.751948] env[63273]: DEBUG nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1332.751948] env[63273]: DEBUG nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1332.752087] env[63273]: DEBUG nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1332.752221] env[63273]: DEBUG nova.network.neutron [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1332.867623] env[63273]: DEBUG neutronclient.v2_0.client [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=63273) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1332.869827] env[63273]: ERROR nova.compute.manager [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Failed to deallocate networks: nova.exception.Unauthorized: Not authorized. [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Traceback (most recent call last): [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] result = getattr(controller, method)(*args, **kwargs) [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._get(image_id) [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] resp, body = self.http_client.get(url, headers=header) [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.request(url, 'GET', **kwargs) [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._handle_response(resp) [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise exc.from_response(resp, resp.content) [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] During handling of the above exception, another exception occurred: [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Traceback (most recent call last): [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self.driver.spawn(context, instance, image_meta, [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._fetch_image_if_missing(context, vi) [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] image_fetch(context, vi, tmp_image_ds_loc) [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] images.fetch_image( [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] metadata = IMAGE_API.get(context, image_ref) [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1332.869827] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return session.show(context, image_id, [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] _reraise_translated_image_exception(image_id) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise new_exc.with_traceback(exc_trace) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] result = getattr(controller, method)(*args, **kwargs) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._get(image_id) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] resp, body = self.http_client.get(url, headers=header) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.request(url, 'GET', **kwargs) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self._handle_response(resp) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise exc.from_response(resp, resp.content) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] During handling of the above exception, another exception occurred: [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Traceback (most recent call last): [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 2448, in _do_build_and_run_instance [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._build_and_run_instance(context, instance, image, [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 2740, in _build_and_run_instance [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise exception.RescheduledException( [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] nova.exception.RescheduledException: Build of instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 was re-scheduled: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] During handling of the above exception, another exception occurred: [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Traceback (most recent call last): [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] exception_handler_v20(status_code, error_body) [ 1332.870911] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise client_exc(message=error_message, [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Neutron server returns request_ids: ['req-b8d24e5b-a732-460f-8c87-5124ed2c7685'] [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] During handling of the above exception, another exception occurred: [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Traceback (most recent call last): [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 3037, in _cleanup_allocated_networks [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._deallocate_network(context, instance, requested_networks) [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self.network_api.deallocate_for_instance( [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] data = neutron.list_ports(**search_opts) [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.list('ports', self.ports_path, retrieve_all, [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] for r in self._pagination(collection, path, **params): [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] res = self.get(path, params=params) [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.retry_request("GET", action, body=body, [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.do_request(method, action, body=body, [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._handle_fault_response(status_code, replybody, resp) [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 204, in wrapper [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise exception.Unauthorized() [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] nova.exception.Unauthorized: Not authorized. [ 1332.871861] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1332.936596] env[63273]: INFO nova.scheduler.client.report [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Deleted allocations for instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 [ 1332.957858] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5d40a512-c827-4054-9382-aeeb7c4b8609 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 630.088s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1332.959059] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 433.748s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1332.959272] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1332.959481] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1332.959689] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1332.962021] env[63273]: INFO nova.compute.manager [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Terminating instance [ 1332.963746] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquiring lock "refresh_cache-d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1332.964034] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Acquired lock "refresh_cache-d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1332.964261] env[63273]: DEBUG nova.network.neutron [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1332.977730] env[63273]: DEBUG nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1333.042779] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1333.079498] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed91d587-c9af-42da-a03c-606288481820 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.089622] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9cb13e1e-a969-4ce3-9a8b-18f7a211eab0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.121234] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-637b4e53-5fd3-42ee-971b-67bcfa5f6d11 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.129056] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52fe818c-22b7-49b0-b8c4-4b1701a80e6c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.142644] env[63273]: DEBUG nova.compute.provider_tree [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1333.152795] env[63273]: DEBUG nova.scheduler.client.report [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1333.169114] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.423s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1333.169619] env[63273]: DEBUG nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1333.172177] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.130s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1333.173644] env[63273]: INFO nova.compute.claims [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1333.204321] env[63273]: DEBUG nova.compute.utils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1333.208195] env[63273]: DEBUG nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1333.208502] env[63273]: DEBUG nova.network.neutron [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1333.224024] env[63273]: DEBUG nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1333.320674] env[63273]: DEBUG nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1333.344601] env[63273]: DEBUG nova.policy [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'bcadcf265aab4f178c48d945073b3934', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '702825b6bec4497f89ef4091bd7af0da', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1333.358439] env[63273]: DEBUG nova.virt.hardware [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1333.358672] env[63273]: DEBUG nova.virt.hardware [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1333.358841] env[63273]: DEBUG nova.virt.hardware [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1333.359069] env[63273]: DEBUG nova.virt.hardware [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1333.359242] env[63273]: DEBUG nova.virt.hardware [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1333.359389] env[63273]: DEBUG nova.virt.hardware [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1333.359596] env[63273]: DEBUG nova.virt.hardware [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1333.359753] env[63273]: DEBUG nova.virt.hardware [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1333.360103] env[63273]: DEBUG nova.virt.hardware [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1333.360267] env[63273]: DEBUG nova.virt.hardware [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1333.360511] env[63273]: DEBUG nova.virt.hardware [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1333.361791] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6dc19ca-5b7b-4492-837c-88c7ee491fe6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.379114] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-300b0f15-ec79-4480-b230-9469ac430c4d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.553487] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-300d3f64-a95a-4329-b78a-5c7ef4a8c482 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.561586] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06fe4d17-2e05-40f0-b30d-e94c0c2325d6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.594059] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad39e5e4-5048-4522-b7da-592c6bfb6c37 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.603027] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f2e4987-c5b8-451f-be5f-80b04d4f99c0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.616840] env[63273]: DEBUG nova.compute.provider_tree [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1333.627595] env[63273]: DEBUG nova.scheduler.client.report [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1333.641949] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.470s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1333.642486] env[63273]: DEBUG nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1333.648351] env[63273]: DEBUG nova.network.neutron [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Updating instance_info_cache with network_info: [{"id": "44913323-d361-4095-8324-0cd854d4e790", "address": "fa:16:3e:ed:41:96", "network": {"id": "79370a65-1eef-4dc5-99ad-719f7ddf5e53", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.147", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "3ebfa0b99d284c0081f368c7c447f2dd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f66f8375-4460-4acd-987b-acda72bfcf0d", "external-id": "nsx-vlan-transportzone-533", "segmentation_id": 533, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap44913323-d3", "ovs_interfaceid": "44913323-d361-4095-8324-0cd854d4e790", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1333.666784] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Releasing lock "refresh_cache-d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1333.667228] env[63273]: DEBUG nova.compute.manager [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1333.667428] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1333.668102] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2dea8ba5-8688-4262-9179-dba6f4842c23 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.680024] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9fb561c7-4ed3-4ae3-9fd5-1b4f9cb79824 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.692153] env[63273]: DEBUG nova.compute.utils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1333.697039] env[63273]: DEBUG nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1333.697039] env[63273]: DEBUG nova.network.neutron [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1333.714844] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d5026f2e-856f-46cd-bf8e-4eb1e5ff8476 could not be found. [ 1333.715053] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1333.715238] env[63273]: INFO nova.compute.manager [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1333.715485] env[63273]: DEBUG oslo.service.loopingcall [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1333.715976] env[63273]: DEBUG nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1333.718578] env[63273]: DEBUG nova.compute.manager [-] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1333.718578] env[63273]: DEBUG nova.network.neutron [-] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1333.814722] env[63273]: DEBUG nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1333.840795] env[63273]: DEBUG nova.policy [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '99a5c8745b574b3689e5ff0ac37a06fd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6a6c31a7f8364be284dfcc2d167ed810', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1333.862835] env[63273]: DEBUG nova.virt.hardware [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1333.863104] env[63273]: DEBUG nova.virt.hardware [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1333.863252] env[63273]: DEBUG nova.virt.hardware [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1333.863492] env[63273]: DEBUG nova.virt.hardware [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1333.863576] env[63273]: DEBUG nova.virt.hardware [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1333.863732] env[63273]: DEBUG nova.virt.hardware [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1333.863924] env[63273]: DEBUG nova.virt.hardware [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1333.864129] env[63273]: DEBUG nova.virt.hardware [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1333.864333] env[63273]: DEBUG nova.virt.hardware [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1333.864500] env[63273]: DEBUG nova.virt.hardware [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1333.864629] env[63273]: DEBUG nova.virt.hardware [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1333.865782] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3873c44f-092e-4b0e-b288-d4acb1717065 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.874608] env[63273]: DEBUG neutronclient.v2_0.client [-] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=63273) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1333.874879] env[63273]: ERROR nova.network.neutron [-] Neutron client was not able to generate a valid admin token, please verify Neutron admin credential located in nova.conf: neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall [-] Dynamic interval looping call 'oslo_service.loopingcall.RetryDecorator.__call__.._func' failed: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall exception_handler_v20(status_code, error_body) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall raise client_exc(message=error_message, [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall Neutron server returns request_ids: ['req-1589c4e1-e9bf-40de-a981-9b2d51bab9ca'] [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall During handling of the above exception, another exception occurred: [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall result = func(*self.args, **self.kw) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall result = f(*args, **kwargs) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall self._deallocate_network( [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall self.network_api.deallocate_for_instance( [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall data = neutron.list_ports(**search_opts) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall return self.list('ports', self.ports_path, retrieve_all, [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall for r in self._pagination(collection, path, **params): [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall res = self.get(path, params=params) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall return self.retry_request("GET", action, body=body, [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall return self.do_request(method, action, body=body, [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall self._handle_fault_response(status_code, replybody, resp) [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1333.875587] env[63273]: ERROR oslo.service.loopingcall [ 1333.877068] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5992a48-2b8f-4dc7-9035-b659710d60ad {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1333.882251] env[63273]: ERROR nova.compute.manager [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Failed to deallocate network for instance. Error: Networking client is experiencing an unauthorized exception.: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1333.932043] env[63273]: ERROR nova.compute.manager [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Setting instance vm_state to ERROR: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Traceback (most recent call last): [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] exception_handler_v20(status_code, error_body) [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise client_exc(message=error_message, [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Neutron server returns request_ids: ['req-1589c4e1-e9bf-40de-a981-9b2d51bab9ca'] [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] During handling of the above exception, another exception occurred: [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Traceback (most recent call last): [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 3333, in do_terminate_instance [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._delete_instance(context, instance, bdms) [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 3268, in _delete_instance [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._shutdown_instance(context, instance, bdms) [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 3162, in _shutdown_instance [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._try_deallocate_network(context, instance, requested_networks) [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 3076, in _try_deallocate_network [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] with excutils.save_and_reraise_exception(): [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self.force_reraise() [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise self.value [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 3074, in _try_deallocate_network [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] _deallocate_network_with_retries() [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return evt.wait() [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] result = hub.switch() [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.greenlet.switch() [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] result = func(*self.args, **self.kw) [ 1333.932043] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] result = f(*args, **kwargs) [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._deallocate_network( [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self.network_api.deallocate_for_instance( [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] data = neutron.list_ports(**search_opts) [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.list('ports', self.ports_path, retrieve_all, [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] for r in self._pagination(collection, path, **params): [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] res = self.get(path, params=params) [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.retry_request("GET", action, body=body, [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] return self.do_request(method, action, body=body, [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] ret = obj(*args, **kwargs) [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] self._handle_fault_response(status_code, replybody, resp) [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1333.933225] env[63273]: ERROR nova.compute.manager [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] [ 1333.962486] env[63273]: DEBUG oslo_concurrency.lockutils [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 1.003s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1333.963667] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 169.740s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1333.963878] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] During sync_power_state the instance has a pending task (deleting). Skip. [ 1333.964143] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "d5026f2e-856f-46cd-bf8e-4eb1e5ff8476" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1334.029927] env[63273]: INFO nova.compute.manager [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] [instance: d5026f2e-856f-46cd-bf8e-4eb1e5ff8476] Successfully reverted task state from None on failure for instance. [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server [None req-fb8a4ef5-9b8f-486e-ab9d-8e5db25528f8 tempest-DeleteServersAdminTestJSON-1565319128 tempest-DeleteServersAdminTestJSON-1565319128-project-member] Exception during message handling: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server exception_handler_v20(status_code, error_body) [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server raise client_exc(message=error_message, [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server Neutron server returns request_ids: ['req-1589c4e1-e9bf-40de-a981-9b2d51bab9ca'] [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 65, in wrapped [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw) [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 166, in decorated_function [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 157, in decorated_function [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/utils.py", line 1453, in decorated_function [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 213, in decorated_function [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 203, in decorated_function [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3345, in terminate_instance [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server do_terminate_instance(instance, bdms) [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py", line 412, in inner [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server return f(*args, **kwargs) [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3340, in do_terminate_instance [ 1334.035646] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3333, in do_terminate_instance [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server self._delete_instance(context, instance, bdms) [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3268, in _delete_instance [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server self._shutdown_instance(context, instance, bdms) [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3162, in _shutdown_instance [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server self._try_deallocate_network(context, instance, requested_networks) [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3076, in _try_deallocate_network [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3074, in _try_deallocate_network [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server _deallocate_network_with_retries() [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server return evt.wait() [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server result = hub.switch() [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server return self.greenlet.switch() [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server result = func(*self.args, **self.kw) [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server result = f(*args, **kwargs) [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server self._deallocate_network( [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server self.network_api.deallocate_for_instance( [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server data = neutron.list_ports(**search_opts) [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server return self.list('ports', self.ports_path, retrieve_all, [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server for r in self._pagination(collection, path, **params): [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server res = self.get(path, params=params) [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server return self.retry_request("GET", action, body=body, [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server return self.do_request(method, action, body=body, [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1334.037222] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1334.038740] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1334.038740] env[63273]: ERROR oslo_messaging.rpc.server self._handle_fault_response(status_code, replybody, resp) [ 1334.038740] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1334.038740] env[63273]: ERROR oslo_messaging.rpc.server raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1334.038740] env[63273]: ERROR oslo_messaging.rpc.server nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1334.038740] env[63273]: ERROR oslo_messaging.rpc.server [ 1334.249467] env[63273]: DEBUG nova.network.neutron [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Successfully created port: bd99d84e-edac-4cce-9f2f-3038259a06df {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1334.503981] env[63273]: DEBUG nova.network.neutron [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Successfully created port: 9ef61f33-7992-4718-b3a2-ff3c431fb71e {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1334.873951] env[63273]: DEBUG nova.network.neutron [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Successfully created port: 74c43e0b-fa4a-4029-ba1e-02bd4eba0a85 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1335.588358] env[63273]: DEBUG nova.compute.manager [req-e68bfe85-abd7-485e-960a-adc8a7d8343c req-3ad07a94-7826-4c79-8397-4d1bf3ef0d72 service nova] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Received event network-vif-plugged-9ef61f33-7992-4718-b3a2-ff3c431fb71e {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1335.588616] env[63273]: DEBUG oslo_concurrency.lockutils [req-e68bfe85-abd7-485e-960a-adc8a7d8343c req-3ad07a94-7826-4c79-8397-4d1bf3ef0d72 service nova] Acquiring lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1335.588860] env[63273]: DEBUG oslo_concurrency.lockutils [req-e68bfe85-abd7-485e-960a-adc8a7d8343c req-3ad07a94-7826-4c79-8397-4d1bf3ef0d72 service nova] Lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1335.589054] env[63273]: DEBUG oslo_concurrency.lockutils [req-e68bfe85-abd7-485e-960a-adc8a7d8343c req-3ad07a94-7826-4c79-8397-4d1bf3ef0d72 service nova] Lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1335.589295] env[63273]: DEBUG nova.compute.manager [req-e68bfe85-abd7-485e-960a-adc8a7d8343c req-3ad07a94-7826-4c79-8397-4d1bf3ef0d72 service nova] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] No waiting events found dispatching network-vif-plugged-9ef61f33-7992-4718-b3a2-ff3c431fb71e {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1335.589396] env[63273]: WARNING nova.compute.manager [req-e68bfe85-abd7-485e-960a-adc8a7d8343c req-3ad07a94-7826-4c79-8397-4d1bf3ef0d72 service nova] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Received unexpected event network-vif-plugged-9ef61f33-7992-4718-b3a2-ff3c431fb71e for instance with vm_state building and task_state spawning. [ 1335.811199] env[63273]: DEBUG nova.network.neutron [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Successfully updated port: 9ef61f33-7992-4718-b3a2-ff3c431fb71e {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1335.829521] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquiring lock "refresh_cache-c4c785e1-274c-4713-98e1-9d7e5e8249a0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1335.829665] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquired lock "refresh_cache-c4c785e1-274c-4713-98e1-9d7e5e8249a0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1335.829899] env[63273]: DEBUG nova.network.neutron [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1335.893115] env[63273]: DEBUG nova.network.neutron [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1336.384229] env[63273]: DEBUG nova.network.neutron [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Successfully updated port: bd99d84e-edac-4cce-9f2f-3038259a06df {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1336.500365] env[63273]: DEBUG nova.network.neutron [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Updating instance_info_cache with network_info: [{"id": "9ef61f33-7992-4718-b3a2-ff3c431fb71e", "address": "fa:16:3e:18:cf:9c", "network": {"id": "13487d00-865f-4658-b027-3465de6e61e4", "bridge": "br-int", "label": "tempest-AttachVolumeShelveTestJSON-583400201-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "6a6c31a7f8364be284dfcc2d167ed810", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "1bf71001-973b-4fda-b804-ee6abcd12776", "external-id": "nsx-vlan-transportzone-498", "segmentation_id": 498, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9ef61f33-79", "ovs_interfaceid": "9ef61f33-7992-4718-b3a2-ff3c431fb71e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1336.512597] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Releasing lock "refresh_cache-c4c785e1-274c-4713-98e1-9d7e5e8249a0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1336.512908] env[63273]: DEBUG nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Instance network_info: |[{"id": "9ef61f33-7992-4718-b3a2-ff3c431fb71e", "address": "fa:16:3e:18:cf:9c", "network": {"id": "13487d00-865f-4658-b027-3465de6e61e4", "bridge": "br-int", "label": "tempest-AttachVolumeShelveTestJSON-583400201-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "6a6c31a7f8364be284dfcc2d167ed810", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "1bf71001-973b-4fda-b804-ee6abcd12776", "external-id": "nsx-vlan-transportzone-498", "segmentation_id": 498, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9ef61f33-79", "ovs_interfaceid": "9ef61f33-7992-4718-b3a2-ff3c431fb71e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1336.513364] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:18:cf:9c', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '1bf71001-973b-4fda-b804-ee6abcd12776', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '9ef61f33-7992-4718-b3a2-ff3c431fb71e', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1336.521344] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Creating folder: Project (6a6c31a7f8364be284dfcc2d167ed810). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1336.522023] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-c0662f68-c9e9-45ea-a626-233b5082e1bf {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1336.534299] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Created folder: Project (6a6c31a7f8364be284dfcc2d167ed810) in parent group-v986930. [ 1336.534519] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Creating folder: Instances. Parent ref: group-v987006. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1336.534968] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-ca4a5d7f-cacd-49f2-85cd-d412e189085d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1336.548424] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Created folder: Instances in parent group-v987006. [ 1336.548846] env[63273]: DEBUG oslo.service.loopingcall [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1336.549094] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1336.549371] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-4e2e2488-a314-4ae1-adf7-b230937733b9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1336.570953] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1336.570953] env[63273]: value = "task-5072113" [ 1336.570953] env[63273]: _type = "Task" [ 1336.570953] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1336.579629] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072113, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1337.080987] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072113, 'name': CreateVM_Task, 'duration_secs': 0.336101} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1337.081421] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1337.081876] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1337.082050] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1337.082378] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1337.082633] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-01473509-42d8-47b2-90fe-9ca1a334556d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1337.088132] env[63273]: DEBUG oslo_vmware.api [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Waiting for the task: (returnval){ [ 1337.088132] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5289fca7-1314-e58c-1348-3c600a4ae50b" [ 1337.088132] env[63273]: _type = "Task" [ 1337.088132] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1337.097125] env[63273]: DEBUG oslo_vmware.api [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5289fca7-1314-e58c-1348-3c600a4ae50b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1337.218411] env[63273]: DEBUG nova.network.neutron [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Successfully updated port: 74c43e0b-fa4a-4029-ba1e-02bd4eba0a85 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1337.232224] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "refresh_cache-5780b2d7-f5b8-47be-8e0e-3d881f15cc90" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1337.232373] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquired lock "refresh_cache-5780b2d7-f5b8-47be-8e0e-3d881f15cc90" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1337.232524] env[63273]: DEBUG nova.network.neutron [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1337.278478] env[63273]: DEBUG nova.network.neutron [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1337.598384] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1337.598657] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1337.598915] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1337.688215] env[63273]: DEBUG nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Received event network-changed-9ef61f33-7992-4718-b3a2-ff3c431fb71e {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1337.688418] env[63273]: DEBUG nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Refreshing instance network info cache due to event network-changed-9ef61f33-7992-4718-b3a2-ff3c431fb71e. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1337.688632] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Acquiring lock "refresh_cache-c4c785e1-274c-4713-98e1-9d7e5e8249a0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1337.688801] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Acquired lock "refresh_cache-c4c785e1-274c-4713-98e1-9d7e5e8249a0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1337.688992] env[63273]: DEBUG nova.network.neutron [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Refreshing network info cache for port 9ef61f33-7992-4718-b3a2-ff3c431fb71e {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1337.893656] env[63273]: DEBUG nova.network.neutron [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Updating instance_info_cache with network_info: [{"id": "bd99d84e-edac-4cce-9f2f-3038259a06df", "address": "fa:16:3e:bf:09:ef", "network": {"id": "0a8609bd-89cb-46ec-ba93-a39957df6bd2", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-1750481792", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.253", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "89f807d9-140f-4a6f-8bce-96795f9482ee", "external-id": "nsx-vlan-transportzone-762", "segmentation_id": 762, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbd99d84e-ed", "ovs_interfaceid": "bd99d84e-edac-4cce-9f2f-3038259a06df", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "74c43e0b-fa4a-4029-ba1e-02bd4eba0a85", "address": "fa:16:3e:07:bc:a4", "network": {"id": "7bf251b2-d33b-40be-9737-236d35706da0", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-758434689", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.32", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c8ee8640-3787-4c27-9581-962ddb2be7e5", "external-id": "nsx-vlan-transportzone-224", "segmentation_id": 224, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap74c43e0b-fa", "ovs_interfaceid": "74c43e0b-fa4a-4029-ba1e-02bd4eba0a85", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1337.911442] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Releasing lock "refresh_cache-5780b2d7-f5b8-47be-8e0e-3d881f15cc90" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1337.911781] env[63273]: DEBUG nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Instance network_info: |[{"id": "bd99d84e-edac-4cce-9f2f-3038259a06df", "address": "fa:16:3e:bf:09:ef", "network": {"id": "0a8609bd-89cb-46ec-ba93-a39957df6bd2", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-1750481792", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.253", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "89f807d9-140f-4a6f-8bce-96795f9482ee", "external-id": "nsx-vlan-transportzone-762", "segmentation_id": 762, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbd99d84e-ed", "ovs_interfaceid": "bd99d84e-edac-4cce-9f2f-3038259a06df", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "74c43e0b-fa4a-4029-ba1e-02bd4eba0a85", "address": "fa:16:3e:07:bc:a4", "network": {"id": "7bf251b2-d33b-40be-9737-236d35706da0", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-758434689", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.32", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c8ee8640-3787-4c27-9581-962ddb2be7e5", "external-id": "nsx-vlan-transportzone-224", "segmentation_id": 224, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap74c43e0b-fa", "ovs_interfaceid": "74c43e0b-fa4a-4029-ba1e-02bd4eba0a85", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1337.912270] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:bf:09:ef', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '89f807d9-140f-4a6f-8bce-96795f9482ee', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'bd99d84e-edac-4cce-9f2f-3038259a06df', 'vif_model': 'vmxnet3'}, {'network_name': 'br-int', 'mac_address': 'fa:16:3e:07:bc:a4', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'c8ee8640-3787-4c27-9581-962ddb2be7e5', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '74c43e0b-fa4a-4029-ba1e-02bd4eba0a85', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1337.921897] env[63273]: DEBUG oslo.service.loopingcall [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1337.922399] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1337.922662] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-5c10b5de-dc95-4710-8376-8ce472946a20 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1337.952251] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1337.952251] env[63273]: value = "task-5072114" [ 1337.952251] env[63273]: _type = "Task" [ 1337.952251] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1337.962375] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072114, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1338.097168] env[63273]: DEBUG nova.network.neutron [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Updated VIF entry in instance network info cache for port 9ef61f33-7992-4718-b3a2-ff3c431fb71e. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1338.097610] env[63273]: DEBUG nova.network.neutron [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Updating instance_info_cache with network_info: [{"id": "9ef61f33-7992-4718-b3a2-ff3c431fb71e", "address": "fa:16:3e:18:cf:9c", "network": {"id": "13487d00-865f-4658-b027-3465de6e61e4", "bridge": "br-int", "label": "tempest-AttachVolumeShelveTestJSON-583400201-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "6a6c31a7f8364be284dfcc2d167ed810", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "1bf71001-973b-4fda-b804-ee6abcd12776", "external-id": "nsx-vlan-transportzone-498", "segmentation_id": 498, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9ef61f33-79", "ovs_interfaceid": "9ef61f33-7992-4718-b3a2-ff3c431fb71e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1338.111625] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Releasing lock "refresh_cache-c4c785e1-274c-4713-98e1-9d7e5e8249a0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1338.111625] env[63273]: DEBUG nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Received event network-vif-plugged-bd99d84e-edac-4cce-9f2f-3038259a06df {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1338.111625] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Acquiring lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1338.111625] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1338.111625] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1338.111625] env[63273]: DEBUG nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] No waiting events found dispatching network-vif-plugged-bd99d84e-edac-4cce-9f2f-3038259a06df {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1338.111625] env[63273]: WARNING nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Received unexpected event network-vif-plugged-bd99d84e-edac-4cce-9f2f-3038259a06df for instance with vm_state building and task_state spawning. [ 1338.111625] env[63273]: DEBUG nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Received event network-changed-bd99d84e-edac-4cce-9f2f-3038259a06df {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1338.111625] env[63273]: DEBUG nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Refreshing instance network info cache due to event network-changed-bd99d84e-edac-4cce-9f2f-3038259a06df. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1338.111881] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Acquiring lock "refresh_cache-5780b2d7-f5b8-47be-8e0e-3d881f15cc90" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1338.113037] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Acquired lock "refresh_cache-5780b2d7-f5b8-47be-8e0e-3d881f15cc90" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1338.113673] env[63273]: DEBUG nova.network.neutron [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Refreshing network info cache for port bd99d84e-edac-4cce-9f2f-3038259a06df {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1338.394478] env[63273]: DEBUG nova.network.neutron [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Updated VIF entry in instance network info cache for port bd99d84e-edac-4cce-9f2f-3038259a06df. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1338.394904] env[63273]: DEBUG nova.network.neutron [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Updating instance_info_cache with network_info: [{"id": "bd99d84e-edac-4cce-9f2f-3038259a06df", "address": "fa:16:3e:bf:09:ef", "network": {"id": "0a8609bd-89cb-46ec-ba93-a39957df6bd2", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-1750481792", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.253", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "89f807d9-140f-4a6f-8bce-96795f9482ee", "external-id": "nsx-vlan-transportzone-762", "segmentation_id": 762, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbd99d84e-ed", "ovs_interfaceid": "bd99d84e-edac-4cce-9f2f-3038259a06df", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "74c43e0b-fa4a-4029-ba1e-02bd4eba0a85", "address": "fa:16:3e:07:bc:a4", "network": {"id": "7bf251b2-d33b-40be-9737-236d35706da0", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-758434689", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.32", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c8ee8640-3787-4c27-9581-962ddb2be7e5", "external-id": "nsx-vlan-transportzone-224", "segmentation_id": 224, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap74c43e0b-fa", "ovs_interfaceid": "74c43e0b-fa4a-4029-ba1e-02bd4eba0a85", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1338.406462] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Releasing lock "refresh_cache-5780b2d7-f5b8-47be-8e0e-3d881f15cc90" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1338.406806] env[63273]: DEBUG nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Received event network-vif-plugged-74c43e0b-fa4a-4029-ba1e-02bd4eba0a85 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1338.407032] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Acquiring lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1338.407246] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1338.407411] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1338.407580] env[63273]: DEBUG nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] No waiting events found dispatching network-vif-plugged-74c43e0b-fa4a-4029-ba1e-02bd4eba0a85 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1338.407833] env[63273]: WARNING nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Received unexpected event network-vif-plugged-74c43e0b-fa4a-4029-ba1e-02bd4eba0a85 for instance with vm_state building and task_state spawning. [ 1338.408229] env[63273]: DEBUG nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Received event network-changed-74c43e0b-fa4a-4029-ba1e-02bd4eba0a85 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1338.408485] env[63273]: DEBUG nova.compute.manager [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Refreshing instance network info cache due to event network-changed-74c43e0b-fa4a-4029-ba1e-02bd4eba0a85. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1338.408799] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Acquiring lock "refresh_cache-5780b2d7-f5b8-47be-8e0e-3d881f15cc90" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1338.408993] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Acquired lock "refresh_cache-5780b2d7-f5b8-47be-8e0e-3d881f15cc90" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1338.409237] env[63273]: DEBUG nova.network.neutron [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Refreshing network info cache for port 74c43e0b-fa4a-4029-ba1e-02bd4eba0a85 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1338.462532] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072114, 'name': CreateVM_Task, 'duration_secs': 0.345017} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1338.464808] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1338.465567] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1338.465734] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1338.466099] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1338.466617] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-29407c7d-8ba4-46ff-a5cb-a6ea42087363 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.471570] env[63273]: DEBUG oslo_vmware.api [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Waiting for the task: (returnval){ [ 1338.471570] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52d12ee9-ea44-6269-b372-d4291802ae12" [ 1338.471570] env[63273]: _type = "Task" [ 1338.471570] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1338.480111] env[63273]: DEBUG oslo_vmware.api [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52d12ee9-ea44-6269-b372-d4291802ae12, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1338.717527] env[63273]: DEBUG nova.network.neutron [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Updated VIF entry in instance network info cache for port 74c43e0b-fa4a-4029-ba1e-02bd4eba0a85. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1338.718076] env[63273]: DEBUG nova.network.neutron [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Updating instance_info_cache with network_info: [{"id": "bd99d84e-edac-4cce-9f2f-3038259a06df", "address": "fa:16:3e:bf:09:ef", "network": {"id": "0a8609bd-89cb-46ec-ba93-a39957df6bd2", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-1750481792", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.253", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "89f807d9-140f-4a6f-8bce-96795f9482ee", "external-id": "nsx-vlan-transportzone-762", "segmentation_id": 762, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbd99d84e-ed", "ovs_interfaceid": "bd99d84e-edac-4cce-9f2f-3038259a06df", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "74c43e0b-fa4a-4029-ba1e-02bd4eba0a85", "address": "fa:16:3e:07:bc:a4", "network": {"id": "7bf251b2-d33b-40be-9737-236d35706da0", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-758434689", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.32", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "702825b6bec4497f89ef4091bd7af0da", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c8ee8640-3787-4c27-9581-962ddb2be7e5", "external-id": "nsx-vlan-transportzone-224", "segmentation_id": 224, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap74c43e0b-fa", "ovs_interfaceid": "74c43e0b-fa4a-4029-ba1e-02bd4eba0a85", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1338.731028] env[63273]: DEBUG oslo_concurrency.lockutils [req-a2f1e4ca-48ff-403a-a957-8ce46cd21dbf req-712e8753-e39b-43dd-b289-657683cdb7b9 service nova] Releasing lock "refresh_cache-5780b2d7-f5b8-47be-8e0e-3d881f15cc90" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1338.983494] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1338.983584] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1338.985027] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1345.894596] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1345.946542] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquiring lock "efe849d5-9df6-4813-a23b-c805e7eb2456" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1345.946785] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Lock "efe849d5-9df6-4813-a23b-c805e7eb2456" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1348.892028] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1348.892028] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1348.903747] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1348.903992] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1348.904181] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1348.904340] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1348.905786] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f73d2fe-6c92-489e-9cc4-13423107858d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1348.915765] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9af410fb-7031-45cd-a13c-02e10afc46fa {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1348.931635] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1279c9bc-6587-43fb-a556-fd6bcc76dbca {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1348.939637] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3f93f1f-d83d-48ad-ac99-5ce984e5945f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1348.970672] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180544MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1348.970826] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1348.971038] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1349.071047] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 13511c60-50cd-44ed-969e-c5fc29b0125e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1349.071248] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1349.071380] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1349.071504] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b2b1ee46-307f-40fa-8346-394a39a0a99b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1349.071628] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fefdf558-7a73-4bae-b57c-b86963189ddb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1349.071751] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a081322d-4636-4bfc-90f6-bd0c617a09e8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1349.071869] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1349.071990] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1349.072122] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1349.072242] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1349.083688] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance e8ed7780-5b17-4693-8cf3-8f75223fce9a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1349.095589] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7b376dab-bef6-45a7-aead-6d4d0e4a3090 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1349.105962] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fa31b240-8bad-48ba-8339-155dc6acb265 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1349.116063] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7dd30edb-74fc-47c8-8506-40bd1470f0f5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1349.125879] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2e353c90-12e0-49dc-aa45-a916f65d6c97 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1349.135836] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 6b83e614-0581-4d5b-8cc9-e08daa0b738a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1349.145771] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92a8d50b-adce-475e-a6f1-da3645afc20a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1349.155686] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 185173ef-30f6-4ddb-bcae-c6e4436d131e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1349.165356] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1da174da-b4e6-437c-a538-53fc78b4282f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1349.175030] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance efe849d5-9df6-4813-a23b-c805e7eb2456 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1349.175271] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1349.175429] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '60', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '7', 'num_os_type_None': '10', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '2', 'io_workload': '10', 'num_proj_ff52e0a10f1c464f808b6df84185058f': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'num_proj_ad0ab37da3b64969894f8ab378f35ff7': '1', 'num_proj_bc9a82a949b84d2db6b4602db73e0d2e': '1', 'num_task_spawning': '3', 'num_proj_71b2d87fd7ff416db331d00f7375e59b': '2', 'num_proj_6a6c31a7f8364be284dfcc2d167ed810': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1349.366474] env[63273]: DEBUG oslo_concurrency.lockutils [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1349.431326] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-098f5a00-c11a-44f1-a00c-c2bf2a1e38e4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1349.439774] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b18fcaf-0fa7-449f-b9b5-eb080fd56b13 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1349.471689] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b76ea421-daf8-4507-95ae-5069d2213cbd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1349.479898] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a3e37e2-52e3-4f27-8196-8f2e19e505b5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1349.493743] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1349.502461] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1349.516599] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1349.516788] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.546s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1350.517271] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1350.892694] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1351.891682] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1351.892083] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1351.892083] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1351.915520] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1351.915769] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1351.915913] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1351.916066] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1351.916261] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1351.916428] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1351.916565] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1351.916690] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1351.916819] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1351.916951] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1351.917083] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1352.914412] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1353.891891] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1353.892102] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1354.892299] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1356.888558] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1364.564545] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquiring lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1364.566725] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.002s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1371.038329] env[63273]: DEBUG oslo_concurrency.lockutils [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquiring lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1380.276093] env[63273]: WARNING oslo_vmware.rw_handles [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1380.276093] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1380.276093] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1380.276093] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1380.276093] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1380.276093] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1380.276093] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1380.276093] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1380.276093] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1380.276093] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1380.276093] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1380.276093] env[63273]: ERROR oslo_vmware.rw_handles [ 1380.276826] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/d658030f-7971-4232-af01-d16443cc1268/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1380.278756] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1380.279010] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Copying Virtual Disk [datastore1] vmware_temp/d658030f-7971-4232-af01-d16443cc1268/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/d658030f-7971-4232-af01-d16443cc1268/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1380.279342] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-8fb981f9-a889-457f-a936-ad744bf26f43 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1380.288172] env[63273]: DEBUG oslo_vmware.api [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Waiting for the task: (returnval){ [ 1380.288172] env[63273]: value = "task-5072115" [ 1380.288172] env[63273]: _type = "Task" [ 1380.288172] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1380.296679] env[63273]: DEBUG oslo_vmware.api [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Task: {'id': task-5072115, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1380.798341] env[63273]: DEBUG oslo_vmware.exceptions [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1380.798637] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1380.799235] env[63273]: ERROR nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1380.799235] env[63273]: Faults: ['InvalidArgument'] [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Traceback (most recent call last): [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] yield resources [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] self.driver.spawn(context, instance, image_meta, [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] self._fetch_image_if_missing(context, vi) [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] image_cache(vi, tmp_image_ds_loc) [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] vm_util.copy_virtual_disk( [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] session._wait_for_task(vmdk_copy_task) [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] return self.wait_for_task(task_ref) [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] return evt.wait() [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] result = hub.switch() [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] return self.greenlet.switch() [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] self.f(*self.args, **self.kw) [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] raise exceptions.translate_fault(task_info.error) [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Faults: ['InvalidArgument'] [ 1380.799235] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] [ 1380.800383] env[63273]: INFO nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Terminating instance [ 1380.801118] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1380.801330] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1380.801595] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-2e669153-ffe4-4892-b5c7-e034d296e5a5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1380.803818] env[63273]: DEBUG nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1380.804023] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1380.804767] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e5a5569-52b1-4a13-b352-142251c65cf5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1380.811880] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1380.812113] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-41540e06-95bc-4e57-afbb-557c84d3e6e8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1380.814542] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1380.814714] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1380.815553] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-9fe5de97-2f6b-46eb-af74-3e14f96bd610 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1380.820401] env[63273]: DEBUG oslo_vmware.api [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Waiting for the task: (returnval){ [ 1380.820401] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e26b8b-b64c-feb6-9a94-2acd47993d25" [ 1380.820401] env[63273]: _type = "Task" [ 1380.820401] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1380.828164] env[63273]: DEBUG oslo_vmware.api [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e26b8b-b64c-feb6-9a94-2acd47993d25, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1380.949094] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1380.949405] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1380.949601] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Deleting the datastore file [datastore1] 13511c60-50cd-44ed-969e-c5fc29b0125e {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1380.949876] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-3392f4c8-2f54-4722-a054-ca35d8fc1c42 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1380.959146] env[63273]: DEBUG oslo_vmware.api [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Waiting for the task: (returnval){ [ 1380.959146] env[63273]: value = "task-5072117" [ 1380.959146] env[63273]: _type = "Task" [ 1380.959146] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1380.967637] env[63273]: DEBUG oslo_vmware.api [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Task: {'id': task-5072117, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1381.331428] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1381.331896] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Creating directory with path [datastore1] vmware_temp/1d9d34f1-75a0-45ad-b337-5e876cab4ad3/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1381.331964] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0815280e-d21a-40f9-81bd-9c20daff0f18 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1381.343855] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Created directory with path [datastore1] vmware_temp/1d9d34f1-75a0-45ad-b337-5e876cab4ad3/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1381.344063] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Fetch image to [datastore1] vmware_temp/1d9d34f1-75a0-45ad-b337-5e876cab4ad3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1381.344242] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/1d9d34f1-75a0-45ad-b337-5e876cab4ad3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1381.345025] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2f50999-ef1a-4894-842d-b62df6c4cd24 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1381.351955] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ec7a47e-72bb-45f8-be1a-e9a79c2f1919 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1381.362454] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7042b79c-495c-4c19-adc7-258e48d1ee4c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1381.395328] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1231aca3-abba-42e4-9de9-694014479301 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1381.401383] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-8cd31c22-1999-4561-8cad-0a3956e6a0a3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1381.423280] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1381.470028] env[63273]: DEBUG oslo_vmware.api [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Task: {'id': task-5072117, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.095618} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1381.470404] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1381.470636] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1381.470818] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1381.470991] env[63273]: INFO nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Took 0.67 seconds to destroy the instance on the hypervisor. [ 1381.473651] env[63273]: DEBUG nova.compute.claims [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1381.473651] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1381.473651] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1381.477192] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1d9d34f1-75a0-45ad-b337-5e876cab4ad3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1381.537116] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1381.537116] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1d9d34f1-75a0-45ad-b337-5e876cab4ad3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1381.793624] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b72753db-7c67-4f2b-a383-b0b9ca826df8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1381.803446] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ad51311-f434-41be-abbf-08282f1e2e95 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1381.835307] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ec3b02a-4704-4083-8911-b08783756ebc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1381.843169] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1aa5b191-2600-4102-9946-ed01a81c3491 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1381.857157] env[63273]: DEBUG nova.compute.provider_tree [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1381.865869] env[63273]: DEBUG nova.scheduler.client.report [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1381.883221] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.409s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1381.883819] env[63273]: ERROR nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1381.883819] env[63273]: Faults: ['InvalidArgument'] [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Traceback (most recent call last): [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] self.driver.spawn(context, instance, image_meta, [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] self._fetch_image_if_missing(context, vi) [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] image_cache(vi, tmp_image_ds_loc) [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] vm_util.copy_virtual_disk( [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] session._wait_for_task(vmdk_copy_task) [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] return self.wait_for_task(task_ref) [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] return evt.wait() [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] result = hub.switch() [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] return self.greenlet.switch() [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] self.f(*self.args, **self.kw) [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] raise exceptions.translate_fault(task_info.error) [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Faults: ['InvalidArgument'] [ 1381.883819] env[63273]: ERROR nova.compute.manager [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] [ 1381.884728] env[63273]: DEBUG nova.compute.utils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1381.886322] env[63273]: DEBUG nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Build of instance 13511c60-50cd-44ed-969e-c5fc29b0125e was re-scheduled: A specified parameter was not correct: fileType [ 1381.886322] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1381.886734] env[63273]: DEBUG nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1381.886919] env[63273]: DEBUG nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1381.887098] env[63273]: DEBUG nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1381.887289] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1382.804547] env[63273]: DEBUG nova.network.neutron [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1382.818032] env[63273]: INFO nova.compute.manager [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Took 0.93 seconds to deallocate network for instance. [ 1382.918045] env[63273]: INFO nova.scheduler.client.report [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Deleted allocations for instance 13511c60-50cd-44ed-969e-c5fc29b0125e [ 1382.942761] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0cffce92-e169-4f38-a6d2-3a0567718a6e tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 624.044s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1382.943981] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 428.505s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1382.944225] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1382.944515] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1382.944705] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1382.947393] env[63273]: INFO nova.compute.manager [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Terminating instance [ 1382.949758] env[63273]: DEBUG nova.compute.manager [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1382.950034] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1382.950638] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1b6d3006-f6ea-48d7-aca8-142ac5de483f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1382.961867] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5828c22-da1a-4e60-8e3f-2ef68717aae6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1382.972161] env[63273]: DEBUG nova.compute.manager [None req-40bebdcc-6777-4a5a-abef-d5f9b678fc8d tempest-ServersNegativeTestMultiTenantJSON-717800779 tempest-ServersNegativeTestMultiTenantJSON-717800779-project-member] [instance: e8ed7780-5b17-4693-8cf3-8f75223fce9a] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1382.999334] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 13511c60-50cd-44ed-969e-c5fc29b0125e could not be found. [ 1382.999569] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1382.999750] env[63273]: INFO nova.compute.manager [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1382.999998] env[63273]: DEBUG oslo.service.loopingcall [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1383.000284] env[63273]: DEBUG nova.compute.manager [-] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1383.000400] env[63273]: DEBUG nova.network.neutron [-] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1383.002810] env[63273]: DEBUG nova.compute.manager [None req-40bebdcc-6777-4a5a-abef-d5f9b678fc8d tempest-ServersNegativeTestMultiTenantJSON-717800779 tempest-ServersNegativeTestMultiTenantJSON-717800779-project-member] [instance: e8ed7780-5b17-4693-8cf3-8f75223fce9a] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1383.024163] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40bebdcc-6777-4a5a-abef-d5f9b678fc8d tempest-ServersNegativeTestMultiTenantJSON-717800779 tempest-ServersNegativeTestMultiTenantJSON-717800779-project-member] Lock "e8ed7780-5b17-4693-8cf3-8f75223fce9a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 207.095s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1383.027956] env[63273]: DEBUG nova.network.neutron [-] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1383.034728] env[63273]: DEBUG nova.compute.manager [None req-ac9bcd13-2ebb-4f4a-b6a0-989a6c751a86 tempest-ServerTagsTestJSON-853966697 tempest-ServerTagsTestJSON-853966697-project-member] [instance: 7b376dab-bef6-45a7-aead-6d4d0e4a3090] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1383.037103] env[63273]: INFO nova.compute.manager [-] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] Took 0.04 seconds to deallocate network for instance. [ 1383.057060] env[63273]: DEBUG nova.compute.manager [None req-ac9bcd13-2ebb-4f4a-b6a0-989a6c751a86 tempest-ServerTagsTestJSON-853966697 tempest-ServerTagsTestJSON-853966697-project-member] [instance: 7b376dab-bef6-45a7-aead-6d4d0e4a3090] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1383.079338] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ac9bcd13-2ebb-4f4a-b6a0-989a6c751a86 tempest-ServerTagsTestJSON-853966697 tempest-ServerTagsTestJSON-853966697-project-member] Lock "7b376dab-bef6-45a7-aead-6d4d0e4a3090" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 200.971s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1383.090425] env[63273]: DEBUG nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1383.127919] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d9045468-5378-48e3-a8bc-d4095c412b60 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.184s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1383.128906] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 218.905s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1383.129112] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 13511c60-50cd-44ed-969e-c5fc29b0125e] During sync_power_state the instance has a pending task (deleting). Skip. [ 1383.129338] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "13511c60-50cd-44ed-969e-c5fc29b0125e" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1383.142040] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1383.142284] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1383.143883] env[63273]: INFO nova.compute.claims [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1383.429825] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34093c91-4b5d-4c42-acb0-8d021a41220b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1383.437548] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc044e2f-8a7b-4b8b-83bb-23d6ca54eba4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1383.468107] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4df21014-eeca-434b-9ee0-4ed3cf320e29 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1383.475797] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bcb13f50-bfe6-44cf-ad6f-25c820968fd4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1383.489932] env[63273]: DEBUG nova.compute.provider_tree [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1383.498530] env[63273]: DEBUG nova.scheduler.client.report [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1383.514044] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.371s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1383.514397] env[63273]: DEBUG nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1383.554956] env[63273]: DEBUG nova.compute.utils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1383.556317] env[63273]: DEBUG nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1383.556494] env[63273]: DEBUG nova.network.neutron [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1383.582016] env[63273]: DEBUG nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1383.633899] env[63273]: DEBUG nova.policy [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e1ab27df058544c496bd2a078b84e6e8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3387519998ea4ed8a586caeaafed2446', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1383.660571] env[63273]: DEBUG nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1383.685998] env[63273]: DEBUG nova.virt.hardware [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1383.686252] env[63273]: DEBUG nova.virt.hardware [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1383.686437] env[63273]: DEBUG nova.virt.hardware [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1383.686627] env[63273]: DEBUG nova.virt.hardware [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1383.686775] env[63273]: DEBUG nova.virt.hardware [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1383.686921] env[63273]: DEBUG nova.virt.hardware [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1383.687152] env[63273]: DEBUG nova.virt.hardware [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1383.687315] env[63273]: DEBUG nova.virt.hardware [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1383.687484] env[63273]: DEBUG nova.virt.hardware [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1383.687647] env[63273]: DEBUG nova.virt.hardware [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1383.687819] env[63273]: DEBUG nova.virt.hardware [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1383.688691] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40836900-097b-40c3-9e5e-a9c5b92ff7ae {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1383.698085] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c174e841-0e69-4b6e-9c89-bcbda39620dd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1383.985716] env[63273]: DEBUG nova.network.neutron [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Successfully created port: 21a630d8-cce3-469f-898b-394a553c3b8a {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1384.719832] env[63273]: DEBUG nova.network.neutron [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Successfully updated port: 21a630d8-cce3-469f-898b-394a553c3b8a {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1384.730863] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "refresh_cache-fa31b240-8bad-48ba-8339-155dc6acb265" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1384.730863] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired lock "refresh_cache-fa31b240-8bad-48ba-8339-155dc6acb265" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1384.730863] env[63273]: DEBUG nova.network.neutron [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1384.774758] env[63273]: DEBUG nova.network.neutron [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1385.041130] env[63273]: DEBUG nova.network.neutron [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Updating instance_info_cache with network_info: [{"id": "21a630d8-cce3-469f-898b-394a553c3b8a", "address": "fa:16:3e:05:57:e5", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap21a630d8-cc", "ovs_interfaceid": "21a630d8-cce3-469f-898b-394a553c3b8a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1385.054998] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Releasing lock "refresh_cache-fa31b240-8bad-48ba-8339-155dc6acb265" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1385.054998] env[63273]: DEBUG nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Instance network_info: |[{"id": "21a630d8-cce3-469f-898b-394a553c3b8a", "address": "fa:16:3e:05:57:e5", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap21a630d8-cc", "ovs_interfaceid": "21a630d8-cce3-469f-898b-394a553c3b8a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1385.055870] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:05:57:e5', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'dad4f433-bb0b-45c7-8040-972ef2277f75', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '21a630d8-cce3-469f-898b-394a553c3b8a', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1385.063012] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Creating folder: Project (3387519998ea4ed8a586caeaafed2446). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1385.063561] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-fd5559a5-6a32-4554-87df-0d25618e5fbc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1385.074907] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Created folder: Project (3387519998ea4ed8a586caeaafed2446) in parent group-v986930. [ 1385.075118] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Creating folder: Instances. Parent ref: group-v987010. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1385.075357] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-4a05cb60-0cdc-4b98-b3b5-0203f284b49c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1385.085078] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Created folder: Instances in parent group-v987010. [ 1385.085333] env[63273]: DEBUG oslo.service.loopingcall [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1385.085524] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1385.085752] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-74cacab1-16cd-4540-ac00-673af5f8198a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1385.106295] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1385.106295] env[63273]: value = "task-5072120" [ 1385.106295] env[63273]: _type = "Task" [ 1385.106295] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1385.114607] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072120, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1385.617027] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072120, 'name': CreateVM_Task, 'duration_secs': 0.318558} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1385.617234] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1385.617934] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1385.618117] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1385.618450] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1385.618710] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-37c02a55-fd29-418b-801b-943f60e47d59 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1385.623488] env[63273]: DEBUG oslo_vmware.api [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 1385.623488] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]529221c0-72af-e7ce-1d74-1a4845a0b4e5" [ 1385.623488] env[63273]: _type = "Task" [ 1385.623488] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1385.631434] env[63273]: DEBUG oslo_vmware.api [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]529221c0-72af-e7ce-1d74-1a4845a0b4e5, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1386.136237] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1386.136528] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1386.137586] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1386.553340] env[63273]: DEBUG nova.compute.manager [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Received event network-vif-plugged-21a630d8-cce3-469f-898b-394a553c3b8a {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1386.553571] env[63273]: DEBUG oslo_concurrency.lockutils [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] Acquiring lock "fa31b240-8bad-48ba-8339-155dc6acb265-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1386.553806] env[63273]: DEBUG oslo_concurrency.lockutils [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] Lock "fa31b240-8bad-48ba-8339-155dc6acb265-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1386.554016] env[63273]: DEBUG oslo_concurrency.lockutils [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] Lock "fa31b240-8bad-48ba-8339-155dc6acb265-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1386.554275] env[63273]: DEBUG nova.compute.manager [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] No waiting events found dispatching network-vif-plugged-21a630d8-cce3-469f-898b-394a553c3b8a {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1386.554457] env[63273]: WARNING nova.compute.manager [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Received unexpected event network-vif-plugged-21a630d8-cce3-469f-898b-394a553c3b8a for instance with vm_state building and task_state spawning. [ 1386.554682] env[63273]: DEBUG nova.compute.manager [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Received event network-changed-21a630d8-cce3-469f-898b-394a553c3b8a {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1386.554881] env[63273]: DEBUG nova.compute.manager [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Refreshing instance network info cache due to event network-changed-21a630d8-cce3-469f-898b-394a553c3b8a. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1386.555147] env[63273]: DEBUG oslo_concurrency.lockutils [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] Acquiring lock "refresh_cache-fa31b240-8bad-48ba-8339-155dc6acb265" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1386.555312] env[63273]: DEBUG oslo_concurrency.lockutils [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] Acquired lock "refresh_cache-fa31b240-8bad-48ba-8339-155dc6acb265" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1386.555476] env[63273]: DEBUG nova.network.neutron [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Refreshing network info cache for port 21a630d8-cce3-469f-898b-394a553c3b8a {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1386.814389] env[63273]: DEBUG nova.network.neutron [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Updated VIF entry in instance network info cache for port 21a630d8-cce3-469f-898b-394a553c3b8a. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1386.814389] env[63273]: DEBUG nova.network.neutron [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Updating instance_info_cache with network_info: [{"id": "21a630d8-cce3-469f-898b-394a553c3b8a", "address": "fa:16:3e:05:57:e5", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap21a630d8-cc", "ovs_interfaceid": "21a630d8-cce3-469f-898b-394a553c3b8a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1386.824958] env[63273]: DEBUG oslo_concurrency.lockutils [req-7c402bd1-5af1-4d11-8d31-dd95725e5754 req-5f388d57-39f2-4466-93d7-761c3d6dfbfc service nova] Releasing lock "refresh_cache-fa31b240-8bad-48ba-8339-155dc6acb265" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1402.542866] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "fa31b240-8bad-48ba-8339-155dc6acb265" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1404.437372] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquiring lock "61bd6d66-590f-4512-afbf-9abf4a308749" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1404.437670] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "61bd6d66-590f-4512-afbf-9abf4a308749" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1407.891586] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1409.891875] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1409.903695] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1409.903926] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1409.904118] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1409.904279] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1409.905383] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6041b469-041f-4a98-a605-88bd5abd1b23 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1409.914756] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-023d3051-186c-490c-8b6b-4898c3e11f64 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1409.929690] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37fb6808-aabb-4d6d-aa37-daf776c8c022 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1409.937417] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96928cd1-1ecf-4bac-a853-46c417a10c74 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1409.968512] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180554MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1409.968955] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1409.970041] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1410.052749] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1410.053342] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1410.053342] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b2b1ee46-307f-40fa-8346-394a39a0a99b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1410.053342] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fefdf558-7a73-4bae-b57c-b86963189ddb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1410.053342] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a081322d-4636-4bfc-90f6-bd0c617a09e8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1410.053571] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1410.053571] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1410.053715] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1410.054100] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1410.054100] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fa31b240-8bad-48ba-8339-155dc6acb265 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1410.065754] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92a8d50b-adce-475e-a6f1-da3645afc20a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1410.079432] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 185173ef-30f6-4ddb-bcae-c6e4436d131e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1410.090890] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1da174da-b4e6-437c-a538-53fc78b4282f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1410.105650] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance efe849d5-9df6-4813-a23b-c805e7eb2456 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1410.116422] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1410.127419] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 61bd6d66-590f-4512-afbf-9abf4a308749 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1410.127658] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1410.127821] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '63', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '9', 'num_os_type_None': '10', 'num_proj_ff52e0a10f1c464f808b6df84185058f': '1', 'io_workload': '10', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'num_proj_ad0ab37da3b64969894f8ab378f35ff7': '1', 'num_proj_bc9a82a949b84d2db6b4602db73e0d2e': '1', 'num_task_spawning': '1', 'num_proj_71b2d87fd7ff416db331d00f7375e59b': '2', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1', 'num_proj_6a6c31a7f8364be284dfcc2d167ed810': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1410.364725] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-109674de-e05c-46ba-9125-534d16750643 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1410.372832] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d8085bd-e730-4fa0-a5b7-ad596fbb39ab {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1410.405251] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58de48f9-766a-4b29-b536-0f104050afa7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1410.413405] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d3657b7-d8b4-4f61-a151-aa5d7260bba8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1410.426801] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1410.435502] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1410.451471] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1410.451674] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.482s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1410.892157] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1410.892511] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1411.899138] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1411.899503] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1411.899503] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1411.921675] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1411.921840] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1411.921955] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1411.922092] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1411.922216] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1411.922338] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1411.922456] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1411.922573] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1411.922705] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1411.922815] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1411.922931] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1411.923424] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1411.923604] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1412.892626] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1412.892814] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11307}} [ 1412.901664] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] There are 0 instances to clean {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11316}} [ 1412.901958] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1412.902123] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances with incomplete migration {{(pid=63273) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11345}} [ 1415.911662] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1415.911919] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1416.707022] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1416.707290] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1416.886519] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1416.891219] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1425.162597] env[63273]: DEBUG oslo_concurrency.lockutils [None req-918a1ecb-50e3-42f4-963c-d4a8ba6b8eac tempest-ServerPasswordTestJSON-1267960498 tempest-ServerPasswordTestJSON-1267960498-project-member] Acquiring lock "dd885558-e9a6-490e-96fa-a4d82db45c5c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1425.162597] env[63273]: DEBUG oslo_concurrency.lockutils [None req-918a1ecb-50e3-42f4-963c-d4a8ba6b8eac tempest-ServerPasswordTestJSON-1267960498 tempest-ServerPasswordTestJSON-1267960498-project-member] Lock "dd885558-e9a6-490e-96fa-a4d82db45c5c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1431.000222] env[63273]: WARNING oslo_vmware.rw_handles [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1431.000222] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1431.000222] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1431.000222] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1431.000222] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1431.000222] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1431.000222] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1431.000222] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1431.000222] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1431.000222] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1431.000222] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1431.000222] env[63273]: ERROR oslo_vmware.rw_handles [ 1431.000847] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/1d9d34f1-75a0-45ad-b337-5e876cab4ad3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1431.002722] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1431.002998] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Copying Virtual Disk [datastore1] vmware_temp/1d9d34f1-75a0-45ad-b337-5e876cab4ad3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/1d9d34f1-75a0-45ad-b337-5e876cab4ad3/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1431.003542] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-f82e3492-0c2d-4328-aaae-c387799ab801 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1431.012024] env[63273]: DEBUG oslo_vmware.api [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Waiting for the task: (returnval){ [ 1431.012024] env[63273]: value = "task-5072121" [ 1431.012024] env[63273]: _type = "Task" [ 1431.012024] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1431.023042] env[63273]: DEBUG oslo_vmware.api [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Task: {'id': task-5072121, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1431.522865] env[63273]: DEBUG oslo_vmware.exceptions [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1431.523195] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1431.523755] env[63273]: ERROR nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1431.523755] env[63273]: Faults: ['InvalidArgument'] [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Traceback (most recent call last): [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] yield resources [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self.driver.spawn(context, instance, image_meta, [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self._fetch_image_if_missing(context, vi) [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] image_cache(vi, tmp_image_ds_loc) [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] vm_util.copy_virtual_disk( [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] session._wait_for_task(vmdk_copy_task) [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] return self.wait_for_task(task_ref) [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] return evt.wait() [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] result = hub.switch() [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] return self.greenlet.switch() [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self.f(*self.args, **self.kw) [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] raise exceptions.translate_fault(task_info.error) [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Faults: ['InvalidArgument'] [ 1431.523755] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] [ 1431.524773] env[63273]: INFO nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Terminating instance [ 1431.525805] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1431.526042] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1431.526560] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquiring lock "refresh_cache-4a90a188-5167-41ca-8cc0-cce7e65fa5cb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1431.526707] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquired lock "refresh_cache-4a90a188-5167-41ca-8cc0-cce7e65fa5cb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1431.526875] env[63273]: DEBUG nova.network.neutron [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1431.527849] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-fb88eec4-e534-4532-879a-902d254ee150 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1431.536716] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1431.536905] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1431.538385] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-281e23f5-c08d-4a23-97f2-19f013fe3ab7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1431.547162] env[63273]: DEBUG oslo_vmware.api [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 1431.547162] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]527ed9d4-cc9b-fd3b-70d8-a91c8e210148" [ 1431.547162] env[63273]: _type = "Task" [ 1431.547162] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1431.555395] env[63273]: DEBUG oslo_vmware.api [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]527ed9d4-cc9b-fd3b-70d8-a91c8e210148, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1431.560961] env[63273]: DEBUG nova.network.neutron [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1431.626019] env[63273]: DEBUG nova.network.neutron [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1431.636832] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Releasing lock "refresh_cache-4a90a188-5167-41ca-8cc0-cce7e65fa5cb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1431.637319] env[63273]: DEBUG nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1431.637606] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1431.638751] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8771369d-44de-4412-a382-a4787da98b15 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1431.647077] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1431.647315] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-857bf4bf-aec6-4463-94d6-dcd04797eb50 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1431.689196] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1431.689437] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1431.689626] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Deleting the datastore file [datastore1] 4a90a188-5167-41ca-8cc0-cce7e65fa5cb {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1431.689958] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-80538576-4d41-4d86-bac2-a3f3102b79a3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1431.696380] env[63273]: DEBUG oslo_vmware.api [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Waiting for the task: (returnval){ [ 1431.696380] env[63273]: value = "task-5072123" [ 1431.696380] env[63273]: _type = "Task" [ 1431.696380] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1431.705417] env[63273]: DEBUG oslo_vmware.api [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Task: {'id': task-5072123, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1432.057623] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1432.058067] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating directory with path [datastore1] vmware_temp/f345f453-8f18-497c-867f-e75981cd75dc/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1432.058574] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a6617053-f856-4779-9d4b-50092134c3ae {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.070643] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created directory with path [datastore1] vmware_temp/f345f453-8f18-497c-867f-e75981cd75dc/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1432.070870] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Fetch image to [datastore1] vmware_temp/f345f453-8f18-497c-867f-e75981cd75dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1432.071068] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/f345f453-8f18-497c-867f-e75981cd75dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1432.071837] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eedf0e3d-378e-4023-a2eb-168463bfad20 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.079451] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d34a074-4196-4cfd-90e9-f9b11bffb8bc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.088855] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2acd1d4a-9a60-4010-9465-d65785d916c6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.120106] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-977afecb-810b-47ea-ab41-abf62469f378 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.126539] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-4f1435b7-a9a2-464f-aa56-22941ec85c61 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.149152] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1432.201916] env[63273]: DEBUG oslo_vmware.rw_handles [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f345f453-8f18-497c-867f-e75981cd75dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1432.259049] env[63273]: DEBUG oslo_vmware.api [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Task: {'id': task-5072123, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.035961} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1432.260480] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1432.260749] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1432.260929] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1432.261125] env[63273]: INFO nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1432.261376] env[63273]: DEBUG oslo.service.loopingcall [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1432.261792] env[63273]: DEBUG oslo_vmware.rw_handles [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1432.261952] env[63273]: DEBUG oslo_vmware.rw_handles [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f345f453-8f18-497c-867f-e75981cd75dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1432.262244] env[63273]: DEBUG nova.compute.manager [-] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Skipping network deallocation for instance since networking was not requested. {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2277}} [ 1432.264545] env[63273]: DEBUG nova.compute.claims [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1432.264717] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1432.264931] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1432.515381] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de9f0ef4-29a6-46b5-a57a-2c002e045e3f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.523415] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0136410-afa4-4f32-a3b0-f06b95848b82 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.553048] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-262ca0d5-5e64-4278-9c1b-3ddc838e9d1b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.563056] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ae15c7f-154c-414e-841a-1adc28ba0894 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.582696] env[63273]: DEBUG nova.compute.provider_tree [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1432.593289] env[63273]: DEBUG nova.scheduler.client.report [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1432.610552] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.345s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1432.610853] env[63273]: ERROR nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1432.610853] env[63273]: Faults: ['InvalidArgument'] [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Traceback (most recent call last): [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self.driver.spawn(context, instance, image_meta, [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self._fetch_image_if_missing(context, vi) [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] image_cache(vi, tmp_image_ds_loc) [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] vm_util.copy_virtual_disk( [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] session._wait_for_task(vmdk_copy_task) [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] return self.wait_for_task(task_ref) [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] return evt.wait() [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] result = hub.switch() [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] return self.greenlet.switch() [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self.f(*self.args, **self.kw) [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] raise exceptions.translate_fault(task_info.error) [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Faults: ['InvalidArgument'] [ 1432.610853] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] [ 1432.611714] env[63273]: DEBUG nova.compute.utils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1432.615033] env[63273]: DEBUG nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Build of instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb was re-scheduled: A specified parameter was not correct: fileType [ 1432.615033] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1432.615430] env[63273]: DEBUG nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1432.615658] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquiring lock "refresh_cache-4a90a188-5167-41ca-8cc0-cce7e65fa5cb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1432.615805] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquired lock "refresh_cache-4a90a188-5167-41ca-8cc0-cce7e65fa5cb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1432.615962] env[63273]: DEBUG nova.network.neutron [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1432.645343] env[63273]: DEBUG nova.network.neutron [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1432.716661] env[63273]: DEBUG nova.network.neutron [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1432.725957] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Releasing lock "refresh_cache-4a90a188-5167-41ca-8cc0-cce7e65fa5cb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1432.726205] env[63273]: DEBUG nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1432.726387] env[63273]: DEBUG nova.compute.manager [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Skipping network deallocation for instance since networking was not requested. {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2277}} [ 1432.821718] env[63273]: INFO nova.scheduler.client.report [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Deleted allocations for instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb [ 1432.844501] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f01ecc10-7ba3-4105-bb76-87f1a126b693 tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 622.847s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1432.846013] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 426.438s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1432.846293] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquiring lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1432.846503] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1432.846670] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1432.848691] env[63273]: INFO nova.compute.manager [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Terminating instance [ 1432.850336] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquiring lock "refresh_cache-4a90a188-5167-41ca-8cc0-cce7e65fa5cb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1432.850495] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Acquired lock "refresh_cache-4a90a188-5167-41ca-8cc0-cce7e65fa5cb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1432.850661] env[63273]: DEBUG nova.network.neutron [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1432.856073] env[63273]: DEBUG nova.compute.manager [None req-4d6af161-8417-4ed0-b070-527beae04be5 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 7dd30edb-74fc-47c8-8506-40bd1470f0f5] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1432.880018] env[63273]: DEBUG nova.network.neutron [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1432.888929] env[63273]: DEBUG nova.compute.manager [None req-4d6af161-8417-4ed0-b070-527beae04be5 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 7dd30edb-74fc-47c8-8506-40bd1470f0f5] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1432.910794] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4d6af161-8417-4ed0-b070-527beae04be5 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "7dd30edb-74fc-47c8-8506-40bd1470f0f5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 225.331s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1432.919940] env[63273]: DEBUG nova.compute.manager [None req-14f74b91-389d-42c3-b65d-481a81582242 tempest-ServerAddressesTestJSON-647465110 tempest-ServerAddressesTestJSON-647465110-project-member] [instance: 2e353c90-12e0-49dc-aa45-a916f65d6c97] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1432.949139] env[63273]: DEBUG nova.compute.manager [None req-14f74b91-389d-42c3-b65d-481a81582242 tempest-ServerAddressesTestJSON-647465110 tempest-ServerAddressesTestJSON-647465110-project-member] [instance: 2e353c90-12e0-49dc-aa45-a916f65d6c97] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1432.953786] env[63273]: DEBUG nova.network.neutron [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1432.963585] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Releasing lock "refresh_cache-4a90a188-5167-41ca-8cc0-cce7e65fa5cb" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1432.963978] env[63273]: DEBUG nova.compute.manager [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1432.964196] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1432.964705] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d26f0674-1d9c-4243-bca1-6334df09c9ac {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.972550] env[63273]: DEBUG oslo_concurrency.lockutils [None req-14f74b91-389d-42c3-b65d-481a81582242 tempest-ServerAddressesTestJSON-647465110 tempest-ServerAddressesTestJSON-647465110-project-member] Lock "2e353c90-12e0-49dc-aa45-a916f65d6c97" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 225.321s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1432.975478] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-baf2f7e0-6387-4290-87e6-38b0881da51c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1432.995108] env[63273]: DEBUG nova.compute.manager [None req-df99a5f9-d1d8-4d31-b624-5d4c41d544b4 tempest-ServerActionsV293TestJSON-1119087857 tempest-ServerActionsV293TestJSON-1119087857-project-member] [instance: 6b83e614-0581-4d5b-8cc9-e08daa0b738a] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1433.009054] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4a90a188-5167-41ca-8cc0-cce7e65fa5cb could not be found. [ 1433.009273] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1433.009459] env[63273]: INFO nova.compute.manager [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1433.009708] env[63273]: DEBUG oslo.service.loopingcall [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1433.010270] env[63273]: DEBUG nova.compute.manager [-] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1433.010372] env[63273]: DEBUG nova.network.neutron [-] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1433.023699] env[63273]: DEBUG nova.compute.manager [None req-df99a5f9-d1d8-4d31-b624-5d4c41d544b4 tempest-ServerActionsV293TestJSON-1119087857 tempest-ServerActionsV293TestJSON-1119087857-project-member] [instance: 6b83e614-0581-4d5b-8cc9-e08daa0b738a] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1433.046013] env[63273]: DEBUG oslo_concurrency.lockutils [None req-df99a5f9-d1d8-4d31-b624-5d4c41d544b4 tempest-ServerActionsV293TestJSON-1119087857 tempest-ServerActionsV293TestJSON-1119087857-project-member] Lock "6b83e614-0581-4d5b-8cc9-e08daa0b738a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 222.039s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1433.056783] env[63273]: DEBUG nova.compute.manager [None req-db777207-2a5e-471b-9ca3-154a7b533edb tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 92a8d50b-adce-475e-a6f1-da3645afc20a] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1433.082761] env[63273]: DEBUG nova.compute.manager [None req-db777207-2a5e-471b-9ca3-154a7b533edb tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 92a8d50b-adce-475e-a6f1-da3645afc20a] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1433.106622] env[63273]: DEBUG oslo_concurrency.lockutils [None req-db777207-2a5e-471b-9ca3-154a7b533edb tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "92a8d50b-adce-475e-a6f1-da3645afc20a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 215.686s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1433.116765] env[63273]: DEBUG nova.compute.manager [None req-1a505c25-bccd-40ad-998b-7947d490a9ab tempest-AttachInterfacesUnderV243Test-1148727018 tempest-AttachInterfacesUnderV243Test-1148727018-project-member] [instance: 185173ef-30f6-4ddb-bcae-c6e4436d131e] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1433.120181] env[63273]: DEBUG neutronclient.v2_0.client [-] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=63273) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1433.120415] env[63273]: ERROR nova.network.neutron [-] Neutron client was not able to generate a valid admin token, please verify Neutron admin credential located in nova.conf: neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall [-] Dynamic interval looping call 'oslo_service.loopingcall.RetryDecorator.__call__.._func' failed: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall exception_handler_v20(status_code, error_body) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall raise client_exc(message=error_message, [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall Neutron server returns request_ids: ['req-226be783-25bf-47b2-b16c-3e855a2cacd6'] [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall During handling of the above exception, another exception occurred: [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall result = func(*self.args, **self.kw) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall result = f(*args, **kwargs) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall self._deallocate_network( [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall self.network_api.deallocate_for_instance( [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall data = neutron.list_ports(**search_opts) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall return self.list('ports', self.ports_path, retrieve_all, [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall for r in self._pagination(collection, path, **params): [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall res = self.get(path, params=params) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall return self.retry_request("GET", action, body=body, [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall return self.do_request(method, action, body=body, [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall self._handle_fault_response(status_code, replybody, resp) [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1433.120974] env[63273]: ERROR oslo.service.loopingcall [ 1433.122603] env[63273]: ERROR nova.compute.manager [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Failed to deallocate network for instance. Error: Networking client is experiencing an unauthorized exception.: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1433.144043] env[63273]: DEBUG nova.compute.manager [None req-1a505c25-bccd-40ad-998b-7947d490a9ab tempest-AttachInterfacesUnderV243Test-1148727018 tempest-AttachInterfacesUnderV243Test-1148727018-project-member] [instance: 185173ef-30f6-4ddb-bcae-c6e4436d131e] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1433.154068] env[63273]: ERROR nova.compute.manager [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Setting instance vm_state to ERROR: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Traceback (most recent call last): [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] ret = obj(*args, **kwargs) [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] exception_handler_v20(status_code, error_body) [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] raise client_exc(message=error_message, [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Neutron server returns request_ids: ['req-226be783-25bf-47b2-b16c-3e855a2cacd6'] [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] During handling of the above exception, another exception occurred: [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Traceback (most recent call last): [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/compute/manager.py", line 3333, in do_terminate_instance [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self._delete_instance(context, instance, bdms) [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/compute/manager.py", line 3268, in _delete_instance [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self._shutdown_instance(context, instance, bdms) [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/compute/manager.py", line 3162, in _shutdown_instance [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self._try_deallocate_network(context, instance, requested_networks) [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/compute/manager.py", line 3076, in _try_deallocate_network [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] with excutils.save_and_reraise_exception(): [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self.force_reraise() [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] raise self.value [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/compute/manager.py", line 3074, in _try_deallocate_network [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] _deallocate_network_with_retries() [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] return evt.wait() [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] result = hub.switch() [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] return self.greenlet.switch() [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] result = func(*self.args, **self.kw) [ 1433.154068] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] result = f(*args, **kwargs) [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self._deallocate_network( [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self.network_api.deallocate_for_instance( [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] data = neutron.list_ports(**search_opts) [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] ret = obj(*args, **kwargs) [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] return self.list('ports', self.ports_path, retrieve_all, [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] ret = obj(*args, **kwargs) [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] for r in self._pagination(collection, path, **params): [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] res = self.get(path, params=params) [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] ret = obj(*args, **kwargs) [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] return self.retry_request("GET", action, body=body, [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] ret = obj(*args, **kwargs) [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] return self.do_request(method, action, body=body, [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] ret = obj(*args, **kwargs) [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] self._handle_fault_response(status_code, replybody, resp) [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1433.155315] env[63273]: ERROR nova.compute.manager [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] [ 1433.168033] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1a505c25-bccd-40ad-998b-7947d490a9ab tempest-AttachInterfacesUnderV243Test-1148727018 tempest-AttachInterfacesUnderV243Test-1148727018-project-member] Lock "185173ef-30f6-4ddb-bcae-c6e4436d131e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 210.417s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1433.177681] env[63273]: DEBUG nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1433.181402] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.335s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1433.182545] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 268.958s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1433.182717] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] During sync_power_state the instance has a pending task (deleting). Skip. [ 1433.182891] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "4a90a188-5167-41ca-8cc0-cce7e65fa5cb" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1433.233725] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1433.234012] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1433.235508] env[63273]: INFO nova.compute.claims [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1433.239652] env[63273]: INFO nova.compute.manager [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] [instance: 4a90a188-5167-41ca-8cc0-cce7e65fa5cb] Successfully reverted task state from None on failure for instance. [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server [None req-b8d90e42-0129-44f7-92b4-fa2a287d89ae tempest-ServersAaction247Test-1404887462 tempest-ServersAaction247Test-1404887462-project-member] Exception during message handling: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server exception_handler_v20(status_code, error_body) [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server raise client_exc(message=error_message, [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server Neutron server returns request_ids: ['req-226be783-25bf-47b2-b16c-3e855a2cacd6'] [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 65, in wrapped [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw) [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 166, in decorated_function [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 157, in decorated_function [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/utils.py", line 1453, in decorated_function [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 213, in decorated_function [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 203, in decorated_function [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3345, in terminate_instance [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server do_terminate_instance(instance, bdms) [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py", line 412, in inner [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server return f(*args, **kwargs) [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3340, in do_terminate_instance [ 1433.243033] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3333, in do_terminate_instance [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server self._delete_instance(context, instance, bdms) [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3268, in _delete_instance [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server self._shutdown_instance(context, instance, bdms) [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3162, in _shutdown_instance [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server self._try_deallocate_network(context, instance, requested_networks) [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3076, in _try_deallocate_network [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3074, in _try_deallocate_network [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server _deallocate_network_with_retries() [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server return evt.wait() [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server result = hub.switch() [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server return self.greenlet.switch() [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server result = func(*self.args, **self.kw) [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server result = f(*args, **kwargs) [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server self._deallocate_network( [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server self.network_api.deallocate_for_instance( [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server data = neutron.list_ports(**search_opts) [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server return self.list('ports', self.ports_path, retrieve_all, [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server for r in self._pagination(collection, path, **params): [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server res = self.get(path, params=params) [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server return self.retry_request("GET", action, body=body, [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server return self.do_request(method, action, body=body, [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1433.244567] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1433.246146] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1433.246146] env[63273]: ERROR oslo_messaging.rpc.server self._handle_fault_response(status_code, replybody, resp) [ 1433.246146] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1433.246146] env[63273]: ERROR oslo_messaging.rpc.server raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1433.246146] env[63273]: ERROR oslo_messaging.rpc.server nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1433.246146] env[63273]: ERROR oslo_messaging.rpc.server [ 1433.476872] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acd731fd-038f-4392-aed5-4344df6097ee {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1433.486023] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d61c25c2-6d78-4afd-91fd-ad6da4afae91 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1433.516296] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72627b42-7c53-465c-9eb7-bf9f5b27e31a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1433.524263] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-300061b6-0a0d-477c-90d8-f8180cec367c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1433.538747] env[63273]: DEBUG nova.compute.provider_tree [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1433.547342] env[63273]: DEBUG nova.scheduler.client.report [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1433.560396] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.326s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1433.560882] env[63273]: DEBUG nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1433.600118] env[63273]: DEBUG nova.compute.utils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1433.601439] env[63273]: DEBUG nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1433.601685] env[63273]: DEBUG nova.network.neutron [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1433.611126] env[63273]: DEBUG nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1433.681466] env[63273]: DEBUG nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1433.686367] env[63273]: DEBUG nova.policy [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7648b8a75224424f8226b574da373a87', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '495ae26325184812a744fc95da67038b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1433.707300] env[63273]: DEBUG nova.virt.hardware [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1433.707547] env[63273]: DEBUG nova.virt.hardware [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1433.707704] env[63273]: DEBUG nova.virt.hardware [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1433.707886] env[63273]: DEBUG nova.virt.hardware [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1433.708044] env[63273]: DEBUG nova.virt.hardware [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1433.708192] env[63273]: DEBUG nova.virt.hardware [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1433.708397] env[63273]: DEBUG nova.virt.hardware [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1433.708557] env[63273]: DEBUG nova.virt.hardware [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1433.708719] env[63273]: DEBUG nova.virt.hardware [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1433.708883] env[63273]: DEBUG nova.virt.hardware [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1433.709065] env[63273]: DEBUG nova.virt.hardware [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1433.709986] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dcbe1b81-4ebf-4bc7-a74f-64605c8f53d8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1433.718134] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a651b598-8dce-4b0c-ae7b-eae7c9ec9f18 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1434.067079] env[63273]: DEBUG nova.network.neutron [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Successfully created port: 427d2c9d-b55c-4a43-b99e-a1f7b90d5978 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1434.764755] env[63273]: DEBUG nova.compute.manager [req-ad642bc8-a47f-415c-b0e8-5b84d38b2efb req-e5cf9946-5cd9-4637-bc8d-4bbbd397c2bc service nova] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Received event network-vif-plugged-427d2c9d-b55c-4a43-b99e-a1f7b90d5978 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1434.765032] env[63273]: DEBUG oslo_concurrency.lockutils [req-ad642bc8-a47f-415c-b0e8-5b84d38b2efb req-e5cf9946-5cd9-4637-bc8d-4bbbd397c2bc service nova] Acquiring lock "1da174da-b4e6-437c-a538-53fc78b4282f-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1434.765215] env[63273]: DEBUG oslo_concurrency.lockutils [req-ad642bc8-a47f-415c-b0e8-5b84d38b2efb req-e5cf9946-5cd9-4637-bc8d-4bbbd397c2bc service nova] Lock "1da174da-b4e6-437c-a538-53fc78b4282f-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1434.765404] env[63273]: DEBUG oslo_concurrency.lockutils [req-ad642bc8-a47f-415c-b0e8-5b84d38b2efb req-e5cf9946-5cd9-4637-bc8d-4bbbd397c2bc service nova] Lock "1da174da-b4e6-437c-a538-53fc78b4282f-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1434.765578] env[63273]: DEBUG nova.compute.manager [req-ad642bc8-a47f-415c-b0e8-5b84d38b2efb req-e5cf9946-5cd9-4637-bc8d-4bbbd397c2bc service nova] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] No waiting events found dispatching network-vif-plugged-427d2c9d-b55c-4a43-b99e-a1f7b90d5978 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1434.765746] env[63273]: WARNING nova.compute.manager [req-ad642bc8-a47f-415c-b0e8-5b84d38b2efb req-e5cf9946-5cd9-4637-bc8d-4bbbd397c2bc service nova] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Received unexpected event network-vif-plugged-427d2c9d-b55c-4a43-b99e-a1f7b90d5978 for instance with vm_state building and task_state spawning. [ 1434.847475] env[63273]: DEBUG nova.network.neutron [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Successfully updated port: 427d2c9d-b55c-4a43-b99e-a1f7b90d5978 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1434.859072] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "refresh_cache-1da174da-b4e6-437c-a538-53fc78b4282f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1434.859317] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "refresh_cache-1da174da-b4e6-437c-a538-53fc78b4282f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1434.859386] env[63273]: DEBUG nova.network.neutron [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1434.925796] env[63273]: DEBUG nova.network.neutron [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1435.139252] env[63273]: DEBUG nova.network.neutron [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Updating instance_info_cache with network_info: [{"id": "427d2c9d-b55c-4a43-b99e-a1f7b90d5978", "address": "fa:16:3e:44:84:cf", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap427d2c9d-b5", "ovs_interfaceid": "427d2c9d-b55c-4a43-b99e-a1f7b90d5978", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1435.153592] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "refresh_cache-1da174da-b4e6-437c-a538-53fc78b4282f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1435.153910] env[63273]: DEBUG nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Instance network_info: |[{"id": "427d2c9d-b55c-4a43-b99e-a1f7b90d5978", "address": "fa:16:3e:44:84:cf", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap427d2c9d-b5", "ovs_interfaceid": "427d2c9d-b55c-4a43-b99e-a1f7b90d5978", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1435.154355] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:44:84:cf', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '69e1a5c1-4ae7-409b-8de7-d401684ef60d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '427d2c9d-b55c-4a43-b99e-a1f7b90d5978', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1435.162051] env[63273]: DEBUG oslo.service.loopingcall [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1435.162522] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1435.162745] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-017f5de1-40f7-46f5-a98a-5bc9b389bddd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1435.183391] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1435.183391] env[63273]: value = "task-5072124" [ 1435.183391] env[63273]: _type = "Task" [ 1435.183391] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1435.191857] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072124, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1435.694149] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072124, 'name': CreateVM_Task, 'duration_secs': 0.312936} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1435.694329] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1435.695015] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1435.695188] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1435.695518] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1435.695768] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5bc5c90f-c31b-48e9-a14e-9555e90efad3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1435.700342] env[63273]: DEBUG oslo_vmware.api [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 1435.700342] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e70f6e-6d0f-a35d-871d-7b5da9ec5712" [ 1435.700342] env[63273]: _type = "Task" [ 1435.700342] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1435.709898] env[63273]: DEBUG oslo_vmware.api [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e70f6e-6d0f-a35d-871d-7b5da9ec5712, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1436.211867] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1436.212270] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1436.212395] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1436.795589] env[63273]: DEBUG nova.compute.manager [req-a7930834-c661-48d7-b70b-23ad39d62f55 req-d9323689-8dee-4944-b7f9-7f42e749a553 service nova] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Received event network-changed-427d2c9d-b55c-4a43-b99e-a1f7b90d5978 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1436.795800] env[63273]: DEBUG nova.compute.manager [req-a7930834-c661-48d7-b70b-23ad39d62f55 req-d9323689-8dee-4944-b7f9-7f42e749a553 service nova] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Refreshing instance network info cache due to event network-changed-427d2c9d-b55c-4a43-b99e-a1f7b90d5978. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1436.796028] env[63273]: DEBUG oslo_concurrency.lockutils [req-a7930834-c661-48d7-b70b-23ad39d62f55 req-d9323689-8dee-4944-b7f9-7f42e749a553 service nova] Acquiring lock "refresh_cache-1da174da-b4e6-437c-a538-53fc78b4282f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1436.796261] env[63273]: DEBUG oslo_concurrency.lockutils [req-a7930834-c661-48d7-b70b-23ad39d62f55 req-d9323689-8dee-4944-b7f9-7f42e749a553 service nova] Acquired lock "refresh_cache-1da174da-b4e6-437c-a538-53fc78b4282f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1436.796372] env[63273]: DEBUG nova.network.neutron [req-a7930834-c661-48d7-b70b-23ad39d62f55 req-d9323689-8dee-4944-b7f9-7f42e749a553 service nova] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Refreshing network info cache for port 427d2c9d-b55c-4a43-b99e-a1f7b90d5978 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1437.358881] env[63273]: DEBUG nova.network.neutron [req-a7930834-c661-48d7-b70b-23ad39d62f55 req-d9323689-8dee-4944-b7f9-7f42e749a553 service nova] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Updated VIF entry in instance network info cache for port 427d2c9d-b55c-4a43-b99e-a1f7b90d5978. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1437.359358] env[63273]: DEBUG nova.network.neutron [req-a7930834-c661-48d7-b70b-23ad39d62f55 req-d9323689-8dee-4944-b7f9-7f42e749a553 service nova] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Updating instance_info_cache with network_info: [{"id": "427d2c9d-b55c-4a43-b99e-a1f7b90d5978", "address": "fa:16:3e:44:84:cf", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap427d2c9d-b5", "ovs_interfaceid": "427d2c9d-b55c-4a43-b99e-a1f7b90d5978", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1437.368938] env[63273]: DEBUG oslo_concurrency.lockutils [req-a7930834-c661-48d7-b70b-23ad39d62f55 req-d9323689-8dee-4944-b7f9-7f42e749a553 service nova] Releasing lock "refresh_cache-1da174da-b4e6-437c-a538-53fc78b4282f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1468.892417] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1470.891827] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1470.904746] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1470.904991] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1470.905180] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1470.905375] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1470.906583] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-934dbdd8-9f4d-46db-b29d-b2122fcaa29c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1470.916028] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0f31ed2-233b-40d6-b336-65b883a4c948 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1470.930786] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1abb24fc-3511-4bea-bdf7-75a65b994088 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1470.937743] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9412395c-51c3-4c38-9244-88c0efe9f714 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1470.968369] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180548MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1470.968528] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1470.968718] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1471.143585] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1471.143752] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance b2b1ee46-307f-40fa-8346-394a39a0a99b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1471.143882] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fefdf558-7a73-4bae-b57c-b86963189ddb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1471.144021] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a081322d-4636-4bfc-90f6-bd0c617a09e8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1471.144140] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1471.144260] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1471.144377] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1471.144491] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1471.144608] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fa31b240-8bad-48ba-8339-155dc6acb265 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1471.144719] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1da174da-b4e6-437c-a538-53fc78b4282f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1471.156526] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance efe849d5-9df6-4813-a23b-c805e7eb2456 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1471.167392] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1471.177334] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 61bd6d66-590f-4512-afbf-9abf4a308749 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1471.187251] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1471.196746] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance dd885558-e9a6-490e-96fa-a4d82db45c5c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1471.196960] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1471.197131] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '69', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '8', 'num_os_type_None': '10', 'num_proj_495ae26325184812a744fc95da67038b': '2', 'io_workload': '10', 'num_proj_418e6c5db17d4baf9c4f7cbc229ad07b': '1', 'num_proj_ad0ab37da3b64969894f8ab378f35ff7': '1', 'num_proj_bc9a82a949b84d2db6b4602db73e0d2e': '1', 'num_task_spawning': '2', 'num_proj_71b2d87fd7ff416db331d00f7375e59b': '2', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1', 'num_proj_6a6c31a7f8364be284dfcc2d167ed810': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1471.213811] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Refreshing inventories for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 1471.228805] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Updating ProviderTree inventory for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 1471.229014] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Updating inventory in ProviderTree for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1471.240927] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Refreshing aggregate associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, aggregates: None {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 1471.259799] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Refreshing trait associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NODE,COMPUTE_IMAGE_TYPE_ISO {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 1471.447580] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f7fc7e9-5738-4255-9eaa-6857bed98f78 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1471.455460] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1982bf4-f5ab-4110-b117-0a7162c0f103 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1471.486086] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6fb8c035-534f-4833-bf12-aa24c1c61dc0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1471.493977] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2adf63a5-aa2c-4c4e-8522-bc6f49e67327 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1471.507823] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1471.518184] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1471.532933] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1471.533140] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.564s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1473.528283] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1473.553802] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1473.554028] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1473.890977] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1473.891182] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1473.891308] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1473.912726] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1473.912927] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1473.913084] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1473.913217] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1473.913342] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1473.913462] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1473.913582] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1473.913701] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1473.913822] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1473.913939] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1473.914070] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1473.914663] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1476.892040] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1476.892333] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1476.892436] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1478.887888] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1480.378742] env[63273]: WARNING oslo_vmware.rw_handles [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1480.378742] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1480.378742] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1480.378742] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1480.378742] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1480.378742] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1480.378742] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1480.378742] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1480.378742] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1480.378742] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1480.378742] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1480.378742] env[63273]: ERROR oslo_vmware.rw_handles [ 1480.379383] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/f345f453-8f18-497c-867f-e75981cd75dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1480.381134] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1480.381460] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Copying Virtual Disk [datastore1] vmware_temp/f345f453-8f18-497c-867f-e75981cd75dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/f345f453-8f18-497c-867f-e75981cd75dc/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1480.381742] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-b377d2ae-2e23-49b2-9321-3871d9e1dde3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1480.391100] env[63273]: DEBUG oslo_vmware.api [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 1480.391100] env[63273]: value = "task-5072125" [ 1480.391100] env[63273]: _type = "Task" [ 1480.391100] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1480.400713] env[63273]: DEBUG oslo_vmware.api [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072125, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1480.901908] env[63273]: DEBUG oslo_vmware.exceptions [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1480.902219] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1480.902819] env[63273]: ERROR nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1480.902819] env[63273]: Faults: ['InvalidArgument'] [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Traceback (most recent call last): [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] yield resources [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] self.driver.spawn(context, instance, image_meta, [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] self._fetch_image_if_missing(context, vi) [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] image_cache(vi, tmp_image_ds_loc) [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] vm_util.copy_virtual_disk( [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] session._wait_for_task(vmdk_copy_task) [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] return self.wait_for_task(task_ref) [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] return evt.wait() [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] result = hub.switch() [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] return self.greenlet.switch() [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] self.f(*self.args, **self.kw) [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] raise exceptions.translate_fault(task_info.error) [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Faults: ['InvalidArgument'] [ 1480.902819] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] [ 1480.903788] env[63273]: INFO nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Terminating instance [ 1480.904752] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1480.904959] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1480.905220] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-10848c40-9657-4804-b14a-22a32f88ba0b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1480.907613] env[63273]: DEBUG nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1480.907807] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1480.908542] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5497b8e3-f595-43a3-9c18-bbf0e4a9e799 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1480.915656] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1480.915878] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-ae7fc941-0e97-40a2-a8a5-0496e28a2605 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1480.918090] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1480.918269] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1480.919233] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f88444c3-ce46-4b51-ae2d-4668b3ceead7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1480.925180] env[63273]: DEBUG oslo_vmware.api [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Waiting for the task: (returnval){ [ 1480.925180] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]523b3d5d-80a0-487f-4181-c69a11e14d7f" [ 1480.925180] env[63273]: _type = "Task" [ 1480.925180] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1480.934548] env[63273]: DEBUG oslo_vmware.api [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]523b3d5d-80a0-487f-4181-c69a11e14d7f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1480.988333] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1480.988580] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1480.988742] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleting the datastore file [datastore1] 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1480.988953] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-6767d8ad-0c5c-4639-a83d-197f69ef2701 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1480.995991] env[63273]: DEBUG oslo_vmware.api [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 1480.995991] env[63273]: value = "task-5072127" [ 1480.995991] env[63273]: _type = "Task" [ 1480.995991] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1481.004942] env[63273]: DEBUG oslo_vmware.api [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072127, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1481.436297] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1481.436622] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Creating directory with path [datastore1] vmware_temp/ab455ff0-6ce2-4616-bec6-4aea3bde42fa/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1481.436865] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-b1609dad-be79-4b7c-857f-4422d864d2ac {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.448633] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Created directory with path [datastore1] vmware_temp/ab455ff0-6ce2-4616-bec6-4aea3bde42fa/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1481.448833] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Fetch image to [datastore1] vmware_temp/ab455ff0-6ce2-4616-bec6-4aea3bde42fa/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1481.449011] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/ab455ff0-6ce2-4616-bec6-4aea3bde42fa/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1481.449761] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfa02e6e-2d5c-4b2f-b33e-fc1f035b7017 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.458264] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d805e41-76a6-4c08-84f8-c2c76d4091db {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.467573] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92554f9e-4ba3-4071-919b-a27f61e89726 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.501669] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c9ef321-685f-43aa-bf17-e073b27dfc8b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.509238] env[63273]: DEBUG oslo_vmware.api [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072127, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.069157} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1481.510817] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1481.511011] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1481.511229] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1481.511389] env[63273]: INFO nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1481.513238] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-6ccea298-363d-4d6d-92bb-8e2d8d716e74 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.516847] env[63273]: DEBUG nova.compute.claims [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1481.517490] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1481.517490] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1481.541059] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1481.686052] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1481.687181] env[63273]: ERROR nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Instance failed to spawn: nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Traceback (most recent call last): [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] result = getattr(controller, method)(*args, **kwargs) [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._get(image_id) [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] resp, body = self.http_client.get(url, headers=header) [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.request(url, 'GET', **kwargs) [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._handle_response(resp) [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise exc.from_response(resp, resp.content) [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] During handling of the above exception, another exception occurred: [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Traceback (most recent call last): [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] yield resources [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self.driver.spawn(context, instance, image_meta, [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._fetch_image_if_missing(context, vi) [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] image_fetch(context, vi, tmp_image_ds_loc) [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] images.fetch_image( [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1481.687181] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] metadata = IMAGE_API.get(context, image_ref) [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return session.show(context, image_id, [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] _reraise_translated_image_exception(image_id) [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise new_exc.with_traceback(exc_trace) [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] result = getattr(controller, method)(*args, **kwargs) [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._get(image_id) [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] resp, body = self.http_client.get(url, headers=header) [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.request(url, 'GET', **kwargs) [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._handle_response(resp) [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise exc.from_response(resp, resp.content) [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1481.688235] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1481.688235] env[63273]: INFO nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Terminating instance [ 1481.689753] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1481.689973] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1481.690638] env[63273]: DEBUG nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1481.690845] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1481.691092] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a3082df8-692f-44dd-be30-100db230fc29 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.693877] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da3c9ef7-1eaa-4066-bfa9-23bfea7eb51f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.703687] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1481.703916] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-842b6c67-d603-4d6d-b9dd-6d2fc38b51c2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.706252] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1481.706733] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1481.707470] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-490e45c7-6cf6-4209-af38-c61034ed6082 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.713906] env[63273]: DEBUG oslo_vmware.api [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Waiting for the task: (returnval){ [ 1481.713906] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]528c7b6d-73f7-d15d-e6fd-5c53252bde9b" [ 1481.713906] env[63273]: _type = "Task" [ 1481.713906] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1481.721797] env[63273]: DEBUG oslo_vmware.api [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]528c7b6d-73f7-d15d-e6fd-5c53252bde9b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1481.763140] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5336d1fe-9f7d-4d8c-8702-f028409b64ac {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.771742] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dce251b-dda3-4208-b830-331bca131aa9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.776379] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1481.776594] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1481.776787] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Deleting the datastore file [datastore1] b2b1ee46-307f-40fa-8346-394a39a0a99b {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1481.777540] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-bb72d26a-f4e1-4ed3-aee6-a523b854a093 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.807877] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70229106-3a3c-43ae-a5fa-79bca6c87a42 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.810640] env[63273]: DEBUG oslo_vmware.api [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Waiting for the task: (returnval){ [ 1481.810640] env[63273]: value = "task-5072129" [ 1481.810640] env[63273]: _type = "Task" [ 1481.810640] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1481.817120] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d18390f-f762-41a0-be76-d6b49306bc9e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1481.823619] env[63273]: DEBUG oslo_vmware.api [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Task: {'id': task-5072129, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1481.833513] env[63273]: DEBUG nova.compute.provider_tree [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1481.842491] env[63273]: DEBUG nova.scheduler.client.report [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1481.860082] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.343s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1481.860754] env[63273]: ERROR nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1481.860754] env[63273]: Faults: ['InvalidArgument'] [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Traceback (most recent call last): [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] self.driver.spawn(context, instance, image_meta, [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] self._fetch_image_if_missing(context, vi) [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] image_cache(vi, tmp_image_ds_loc) [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] vm_util.copy_virtual_disk( [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] session._wait_for_task(vmdk_copy_task) [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] return self.wait_for_task(task_ref) [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] return evt.wait() [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] result = hub.switch() [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] return self.greenlet.switch() [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] self.f(*self.args, **self.kw) [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] raise exceptions.translate_fault(task_info.error) [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Faults: ['InvalidArgument'] [ 1481.860754] env[63273]: ERROR nova.compute.manager [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] [ 1481.861608] env[63273]: DEBUG nova.compute.utils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1481.863242] env[63273]: DEBUG nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Build of instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 was re-scheduled: A specified parameter was not correct: fileType [ 1481.863242] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1481.863707] env[63273]: DEBUG nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1481.863883] env[63273]: DEBUG nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1481.864076] env[63273]: DEBUG nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1481.864247] env[63273]: DEBUG nova.network.neutron [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1482.209784] env[63273]: DEBUG nova.network.neutron [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1482.221657] env[63273]: INFO nova.compute.manager [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Took 0.36 seconds to deallocate network for instance. [ 1482.227623] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1482.227833] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Creating directory with path [datastore1] vmware_temp/c873d914-7f34-4348-b735-c9e3fa8103f3/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1482.228092] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-f19aaf8b-b441-44e3-adf7-505b8524509a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.241823] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Created directory with path [datastore1] vmware_temp/c873d914-7f34-4348-b735-c9e3fa8103f3/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1482.242087] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Fetch image to [datastore1] vmware_temp/c873d914-7f34-4348-b735-c9e3fa8103f3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1482.242273] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/c873d914-7f34-4348-b735-c9e3fa8103f3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1482.243077] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5c84dabd-8ec5-4962-8af3-3b4e3ea73197 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.250752] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c950a752-f803-4557-8d8b-b5525ae767fb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.262573] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd016956-be01-4212-b47d-fce470f607c1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.298268] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0bb67a27-f57b-4488-97ef-b057e05475a3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.309097] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-a40ce06d-da93-41d2-9b35-fa94e1c682b6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.320107] env[63273]: DEBUG oslo_vmware.api [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Task: {'id': task-5072129, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.066893} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1482.320258] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1482.320391] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1482.320566] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1482.320737] env[63273]: INFO nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Took 0.63 seconds to destroy the instance on the hypervisor. [ 1482.323325] env[63273]: DEBUG nova.compute.claims [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1482.323325] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1482.323519] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1482.341865] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1482.344784] env[63273]: INFO nova.scheduler.client.report [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleted allocations for instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 [ 1482.391809] env[63273]: DEBUG oslo_concurrency.lockutils [None req-c728c754-6416-4a23-9f42-8d193ddd599e tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 583.272s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1482.392996] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 387.805s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1482.393676] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1482.393912] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1482.394369] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1482.396283] env[63273]: INFO nova.compute.manager [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Terminating instance [ 1482.398238] env[63273]: DEBUG nova.compute.manager [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1482.398438] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1482.398938] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9465bfa8-5dec-4376-ba18-f3521519b47d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.410958] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1baf4287-8a66-47e1-9e89-5e5377bc6b63 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.426998] env[63273]: DEBUG nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1482.460350] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817 could not be found. [ 1482.460712] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1482.460818] env[63273]: INFO nova.compute.manager [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Took 0.06 seconds to destroy the instance on the hypervisor. [ 1482.460975] env[63273]: DEBUG oslo.service.loopingcall [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1482.462160] env[63273]: DEBUG oslo_vmware.rw_handles [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/c873d914-7f34-4348-b735-c9e3fa8103f3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1482.467415] env[63273]: DEBUG nova.compute.manager [-] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1482.467541] env[63273]: DEBUG nova.network.neutron [-] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1482.534031] env[63273]: DEBUG oslo_vmware.rw_handles [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1482.534031] env[63273]: DEBUG oslo_vmware.rw_handles [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/c873d914-7f34-4348-b735-c9e3fa8103f3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1482.543968] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1482.682614] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-82ecfff2-4141-464f-9686-035b156d0bf3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.690574] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ac7e41e-6c44-40ca-8711-b44ae770bbce {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.721880] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4be83f0c-a0a2-4370-89ec-3a0e82270a66 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.730156] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79eafc4e-7b46-443c-a2fc-0f5383cb4f11 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.733960] env[63273]: DEBUG nova.network.neutron [-] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1482.745158] env[63273]: DEBUG nova.compute.provider_tree [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1482.746268] env[63273]: INFO nova.compute.manager [-] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] Took 0.28 seconds to deallocate network for instance. [ 1482.752636] env[63273]: DEBUG nova.scheduler.client.report [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1482.766724] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.443s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1482.767467] env[63273]: ERROR nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Failed to build and run instance: nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Traceback (most recent call last): [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] result = getattr(controller, method)(*args, **kwargs) [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._get(image_id) [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] resp, body = self.http_client.get(url, headers=header) [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.request(url, 'GET', **kwargs) [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._handle_response(resp) [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise exc.from_response(resp, resp.content) [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] During handling of the above exception, another exception occurred: [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Traceback (most recent call last): [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self.driver.spawn(context, instance, image_meta, [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._fetch_image_if_missing(context, vi) [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] image_fetch(context, vi, tmp_image_ds_loc) [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] images.fetch_image( [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] metadata = IMAGE_API.get(context, image_ref) [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1482.767467] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return session.show(context, image_id, [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] _reraise_translated_image_exception(image_id) [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise new_exc.with_traceback(exc_trace) [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] result = getattr(controller, method)(*args, **kwargs) [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._get(image_id) [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] resp, body = self.http_client.get(url, headers=header) [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.request(url, 'GET', **kwargs) [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._handle_response(resp) [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise exc.from_response(resp, resp.content) [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1482.769066] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.769066] env[63273]: DEBUG nova.compute.utils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1482.769769] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.226s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1482.771071] env[63273]: INFO nova.compute.claims [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1482.773713] env[63273]: DEBUG nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Build of instance b2b1ee46-307f-40fa-8346-394a39a0a99b was re-scheduled: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1482.774204] env[63273]: DEBUG nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1482.774395] env[63273]: DEBUG nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1482.774571] env[63273]: DEBUG nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1482.774737] env[63273]: DEBUG nova.network.neutron [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1482.874701] env[63273]: DEBUG oslo_concurrency.lockutils [None req-8c2492d6-ccfb-457a-92fa-4ce23d832ef0 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.481s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1482.875755] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 318.651s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1482.875755] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817] During sync_power_state the instance has a pending task (deleting). Skip. [ 1482.875755] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "0e1bc6e5-b2f8-4a2f-a2d0-9552b876f817" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1482.878336] env[63273]: DEBUG neutronclient.v2_0.client [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=63273) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1482.879542] env[63273]: ERROR nova.compute.manager [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Failed to deallocate networks: nova.exception.Unauthorized: Not authorized. [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Traceback (most recent call last): [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] result = getattr(controller, method)(*args, **kwargs) [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._get(image_id) [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] resp, body = self.http_client.get(url, headers=header) [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.request(url, 'GET', **kwargs) [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._handle_response(resp) [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise exc.from_response(resp, resp.content) [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] During handling of the above exception, another exception occurred: [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Traceback (most recent call last): [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self.driver.spawn(context, instance, image_meta, [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._fetch_image_if_missing(context, vi) [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] image_fetch(context, vi, tmp_image_ds_loc) [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] images.fetch_image( [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] metadata = IMAGE_API.get(context, image_ref) [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1482.879542] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return session.show(context, image_id, [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] _reraise_translated_image_exception(image_id) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise new_exc.with_traceback(exc_trace) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] result = getattr(controller, method)(*args, **kwargs) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._get(image_id) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] resp, body = self.http_client.get(url, headers=header) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.request(url, 'GET', **kwargs) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self._handle_response(resp) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise exc.from_response(resp, resp.content) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] During handling of the above exception, another exception occurred: [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Traceback (most recent call last): [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 2448, in _do_build_and_run_instance [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._build_and_run_instance(context, instance, image, [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 2740, in _build_and_run_instance [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise exception.RescheduledException( [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] nova.exception.RescheduledException: Build of instance b2b1ee46-307f-40fa-8346-394a39a0a99b was re-scheduled: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] During handling of the above exception, another exception occurred: [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Traceback (most recent call last): [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] exception_handler_v20(status_code, error_body) [ 1482.880779] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise client_exc(message=error_message, [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Neutron server returns request_ids: ['req-8dbdcdf3-814c-4e7e-b3cf-65dce75641aa'] [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] During handling of the above exception, another exception occurred: [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Traceback (most recent call last): [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 3037, in _cleanup_allocated_networks [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._deallocate_network(context, instance, requested_networks) [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self.network_api.deallocate_for_instance( [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] data = neutron.list_ports(**search_opts) [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.list('ports', self.ports_path, retrieve_all, [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] for r in self._pagination(collection, path, **params): [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] res = self.get(path, params=params) [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.retry_request("GET", action, body=body, [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.do_request(method, action, body=body, [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._handle_fault_response(status_code, replybody, resp) [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 204, in wrapper [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise exception.Unauthorized() [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] nova.exception.Unauthorized: Not authorized. [ 1482.881914] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1482.936286] env[63273]: INFO nova.scheduler.client.report [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Deleted allocations for instance b2b1ee46-307f-40fa-8346-394a39a0a99b [ 1482.954909] env[63273]: DEBUG oslo_concurrency.lockutils [None req-40260aca-18d2-4bc2-b023-be4bab67b050 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "b2b1ee46-307f-40fa-8346-394a39a0a99b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 534.088s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1482.955997] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "b2b1ee46-307f-40fa-8346-394a39a0a99b" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 338.293s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1482.956225] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Acquiring lock "b2b1ee46-307f-40fa-8346-394a39a0a99b-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1482.956429] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "b2b1ee46-307f-40fa-8346-394a39a0a99b-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1482.956615] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "b2b1ee46-307f-40fa-8346-394a39a0a99b-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1482.958497] env[63273]: INFO nova.compute.manager [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Terminating instance [ 1482.960172] env[63273]: DEBUG nova.compute.manager [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1482.960400] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1482.961175] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b84df13f-9db0-498d-99da-9f3e96190c91 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1482.968041] env[63273]: DEBUG nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1482.974450] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2dcfb99f-38a9-4bcd-9fa4-6d3e0170d2a5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.009283] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b2b1ee46-307f-40fa-8346-394a39a0a99b could not be found. [ 1483.009788] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1483.010078] env[63273]: INFO nova.compute.manager [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1483.010428] env[63273]: DEBUG oslo.service.loopingcall [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1483.017887] env[63273]: DEBUG nova.compute.manager [-] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1483.018124] env[63273]: DEBUG nova.network.neutron [-] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1483.033400] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1483.064263] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5acfc508-dea1-4ff0-8322-2efeb7cb9149 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.073529] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a19081a-d382-42b8-bb8b-b42146d60d0a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.104578] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7714b2f-4d91-47b1-a1fa-c9acf38d670b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.107380] env[63273]: DEBUG neutronclient.v2_0.client [-] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=63273) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1483.107609] env[63273]: ERROR nova.network.neutron [-] Neutron client was not able to generate a valid admin token, please verify Neutron admin credential located in nova.conf: neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall [-] Dynamic interval looping call 'oslo_service.loopingcall.RetryDecorator.__call__.._func' failed: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall exception_handler_v20(status_code, error_body) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall raise client_exc(message=error_message, [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall Neutron server returns request_ids: ['req-36169b24-c214-44e4-bb33-49bfa80c90c9'] [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall During handling of the above exception, another exception occurred: [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall result = func(*self.args, **self.kw) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall result = f(*args, **kwargs) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall self._deallocate_network( [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall self.network_api.deallocate_for_instance( [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall data = neutron.list_ports(**search_opts) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall return self.list('ports', self.ports_path, retrieve_all, [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall for r in self._pagination(collection, path, **params): [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall res = self.get(path, params=params) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall return self.retry_request("GET", action, body=body, [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall return self.do_request(method, action, body=body, [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall self._handle_fault_response(status_code, replybody, resp) [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1483.108135] env[63273]: ERROR oslo.service.loopingcall [ 1483.109640] env[63273]: ERROR nova.compute.manager [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Failed to deallocate network for instance. Error: Networking client is experiencing an unauthorized exception.: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1483.115952] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-021dad33-dbd9-42ab-93d7-3a7a65eef87a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.131300] env[63273]: DEBUG nova.compute.provider_tree [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1483.136919] env[63273]: ERROR nova.compute.manager [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Setting instance vm_state to ERROR: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Traceback (most recent call last): [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] exception_handler_v20(status_code, error_body) [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise client_exc(message=error_message, [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Neutron server returns request_ids: ['req-36169b24-c214-44e4-bb33-49bfa80c90c9'] [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] During handling of the above exception, another exception occurred: [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Traceback (most recent call last): [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 3333, in do_terminate_instance [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._delete_instance(context, instance, bdms) [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 3268, in _delete_instance [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._shutdown_instance(context, instance, bdms) [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 3162, in _shutdown_instance [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._try_deallocate_network(context, instance, requested_networks) [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 3076, in _try_deallocate_network [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] with excutils.save_and_reraise_exception(): [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self.force_reraise() [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise self.value [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 3074, in _try_deallocate_network [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] _deallocate_network_with_retries() [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return evt.wait() [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] result = hub.switch() [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.greenlet.switch() [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] result = func(*self.args, **self.kw) [ 1483.136919] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] result = f(*args, **kwargs) [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._deallocate_network( [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self.network_api.deallocate_for_instance( [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] data = neutron.list_ports(**search_opts) [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.list('ports', self.ports_path, retrieve_all, [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] for r in self._pagination(collection, path, **params): [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] res = self.get(path, params=params) [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.retry_request("GET", action, body=body, [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] return self.do_request(method, action, body=body, [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] ret = obj(*args, **kwargs) [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] self._handle_fault_response(status_code, replybody, resp) [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1483.138180] env[63273]: ERROR nova.compute.manager [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] [ 1483.142014] env[63273]: DEBUG nova.scheduler.client.report [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1483.154861] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.386s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1483.155340] env[63273]: DEBUG nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1483.157544] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.124s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1483.158988] env[63273]: INFO nova.compute.claims [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1483.172574] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Lock "b2b1ee46-307f-40fa-8346-394a39a0a99b" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.217s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1483.173592] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "b2b1ee46-307f-40fa-8346-394a39a0a99b" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 318.949s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1483.174440] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] During sync_power_state the instance has a pending task (deleting). Skip. [ 1483.174440] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "b2b1ee46-307f-40fa-8346-394a39a0a99b" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1483.192718] env[63273]: DEBUG nova.compute.utils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1483.194742] env[63273]: DEBUG nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1483.194923] env[63273]: DEBUG nova.network.neutron [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1483.203514] env[63273]: DEBUG nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1483.240280] env[63273]: INFO nova.compute.manager [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] [instance: b2b1ee46-307f-40fa-8346-394a39a0a99b] Successfully reverted task state from None on failure for instance. [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server [None req-3135d263-14da-47a0-9871-67e826603332 tempest-MigrationsAdminTest-1600285079 tempest-MigrationsAdminTest-1600285079-project-member] Exception during message handling: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server exception_handler_v20(status_code, error_body) [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server raise client_exc(message=error_message, [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server Neutron server returns request_ids: ['req-36169b24-c214-44e4-bb33-49bfa80c90c9'] [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 65, in wrapped [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw) [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 166, in decorated_function [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 157, in decorated_function [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/utils.py", line 1453, in decorated_function [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 213, in decorated_function [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 203, in decorated_function [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3345, in terminate_instance [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server do_terminate_instance(instance, bdms) [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py", line 412, in inner [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server return f(*args, **kwargs) [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3340, in do_terminate_instance [ 1483.247023] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3333, in do_terminate_instance [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server self._delete_instance(context, instance, bdms) [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3268, in _delete_instance [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server self._shutdown_instance(context, instance, bdms) [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3162, in _shutdown_instance [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server self._try_deallocate_network(context, instance, requested_networks) [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3076, in _try_deallocate_network [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3074, in _try_deallocate_network [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server _deallocate_network_with_retries() [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server return evt.wait() [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server result = hub.switch() [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server return self.greenlet.switch() [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server result = func(*self.args, **self.kw) [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server result = f(*args, **kwargs) [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server self._deallocate_network( [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server self.network_api.deallocate_for_instance( [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server data = neutron.list_ports(**search_opts) [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server return self.list('ports', self.ports_path, retrieve_all, [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server for r in self._pagination(collection, path, **params): [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server res = self.get(path, params=params) [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server return self.retry_request("GET", action, body=body, [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server return self.do_request(method, action, body=body, [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1483.249264] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1483.251090] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1483.251090] env[63273]: ERROR oslo_messaging.rpc.server self._handle_fault_response(status_code, replybody, resp) [ 1483.251090] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1483.251090] env[63273]: ERROR oslo_messaging.rpc.server raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1483.251090] env[63273]: ERROR oslo_messaging.rpc.server nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1483.251090] env[63273]: ERROR oslo_messaging.rpc.server [ 1483.267677] env[63273]: DEBUG nova.policy [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7f3d2a576a834a96b0923e76778eaf2e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '72f5eacfc9004694a5107a00edabbdd6', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1483.273299] env[63273]: DEBUG nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1483.300936] env[63273]: DEBUG nova.virt.hardware [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1483.301338] env[63273]: DEBUG nova.virt.hardware [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1483.301602] env[63273]: DEBUG nova.virt.hardware [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1483.301880] env[63273]: DEBUG nova.virt.hardware [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1483.302155] env[63273]: DEBUG nova.virt.hardware [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1483.302376] env[63273]: DEBUG nova.virt.hardware [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1483.302690] env[63273]: DEBUG nova.virt.hardware [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1483.302882] env[63273]: DEBUG nova.virt.hardware [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1483.303250] env[63273]: DEBUG nova.virt.hardware [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1483.303250] env[63273]: DEBUG nova.virt.hardware [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1483.303413] env[63273]: DEBUG nova.virt.hardware [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1483.304395] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15185581-b56a-4caa-945e-4a5a15fd2393 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.315629] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cab32b9b-3620-4d70-9f5e-fe6f14249a16 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.403007] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ac6d775-4ea7-4d01-9865-7f9c30d3f7ed {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.411390] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95e18d92-55b8-4a88-bf08-651b96855659 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.444921] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47afad73-32a2-45ad-8e71-0fdcfc47a0b4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.453259] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b499affb-d4ee-43ab-b617-095c5ea8a95d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.467197] env[63273]: DEBUG nova.compute.provider_tree [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1483.476958] env[63273]: DEBUG nova.scheduler.client.report [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1483.499929] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.342s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1483.500623] env[63273]: DEBUG nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1483.567169] env[63273]: DEBUG nova.compute.utils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1483.572020] env[63273]: DEBUG nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1483.572020] env[63273]: DEBUG nova.network.neutron [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1483.578879] env[63273]: DEBUG nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1483.661385] env[63273]: DEBUG nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1483.687750] env[63273]: DEBUG nova.virt.hardware [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1483.687937] env[63273]: DEBUG nova.virt.hardware [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1483.688116] env[63273]: DEBUG nova.virt.hardware [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1483.688330] env[63273]: DEBUG nova.virt.hardware [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1483.688482] env[63273]: DEBUG nova.virt.hardware [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1483.688629] env[63273]: DEBUG nova.virt.hardware [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1483.688839] env[63273]: DEBUG nova.virt.hardware [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1483.688996] env[63273]: DEBUG nova.virt.hardware [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1483.689181] env[63273]: DEBUG nova.virt.hardware [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1483.689346] env[63273]: DEBUG nova.virt.hardware [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1483.689521] env[63273]: DEBUG nova.virt.hardware [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1483.690439] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1dd4eb7f-c48c-49cb-97cd-507751ecdab4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.694239] env[63273]: DEBUG nova.policy [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6e74edba9b6a40fcb2f166ab4400040b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd277c5d52dd44fbdab7560c6fb34cf31', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1483.701767] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ce14ae0-8065-4d87-853e-1ba0cf89582e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1483.919578] env[63273]: DEBUG nova.network.neutron [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Successfully created port: 85f37247-c66a-4e8e-af09-6520b0d5a973 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1484.364830] env[63273]: DEBUG nova.network.neutron [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Successfully created port: 4a713d74-61c8-4061-856f-83bfe2d702d5 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1484.950639] env[63273]: DEBUG nova.compute.manager [req-4be78347-9d81-4b96-883b-9f57183bab2c req-47e4924c-0e77-4e6e-adc8-cef46603ecc5 service nova] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Received event network-vif-plugged-85f37247-c66a-4e8e-af09-6520b0d5a973 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1484.950944] env[63273]: DEBUG oslo_concurrency.lockutils [req-4be78347-9d81-4b96-883b-9f57183bab2c req-47e4924c-0e77-4e6e-adc8-cef46603ecc5 service nova] Acquiring lock "efe849d5-9df6-4813-a23b-c805e7eb2456-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1484.951105] env[63273]: DEBUG oslo_concurrency.lockutils [req-4be78347-9d81-4b96-883b-9f57183bab2c req-47e4924c-0e77-4e6e-adc8-cef46603ecc5 service nova] Lock "efe849d5-9df6-4813-a23b-c805e7eb2456-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1484.951379] env[63273]: DEBUG oslo_concurrency.lockutils [req-4be78347-9d81-4b96-883b-9f57183bab2c req-47e4924c-0e77-4e6e-adc8-cef46603ecc5 service nova] Lock "efe849d5-9df6-4813-a23b-c805e7eb2456-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1484.951775] env[63273]: DEBUG nova.compute.manager [req-4be78347-9d81-4b96-883b-9f57183bab2c req-47e4924c-0e77-4e6e-adc8-cef46603ecc5 service nova] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] No waiting events found dispatching network-vif-plugged-85f37247-c66a-4e8e-af09-6520b0d5a973 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1484.951871] env[63273]: WARNING nova.compute.manager [req-4be78347-9d81-4b96-883b-9f57183bab2c req-47e4924c-0e77-4e6e-adc8-cef46603ecc5 service nova] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Received unexpected event network-vif-plugged-85f37247-c66a-4e8e-af09-6520b0d5a973 for instance with vm_state building and task_state spawning. [ 1485.139632] env[63273]: DEBUG nova.network.neutron [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Successfully updated port: 85f37247-c66a-4e8e-af09-6520b0d5a973 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1485.152685] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquiring lock "refresh_cache-efe849d5-9df6-4813-a23b-c805e7eb2456" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1485.152933] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquired lock "refresh_cache-efe849d5-9df6-4813-a23b-c805e7eb2456" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1485.152996] env[63273]: DEBUG nova.network.neutron [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1485.254076] env[63273]: DEBUG nova.network.neutron [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1485.599159] env[63273]: DEBUG nova.network.neutron [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Updating instance_info_cache with network_info: [{"id": "85f37247-c66a-4e8e-af09-6520b0d5a973", "address": "fa:16:3e:38:f5:8a", "network": {"id": "58774d83-9d0a-42ae-979d-a9c35214ca47", "bridge": "br-int", "label": "tempest-ServersNegativeTestJSON-2012108761-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "72f5eacfc9004694a5107a00edabbdd6", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "095fbf26-7367-4f9e-87c5-2965b64b0b0f", "external-id": "nsx-vlan-transportzone-777", "segmentation_id": 777, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap85f37247-c6", "ovs_interfaceid": "85f37247-c66a-4e8e-af09-6520b0d5a973", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1485.614847] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Releasing lock "refresh_cache-efe849d5-9df6-4813-a23b-c805e7eb2456" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1485.615162] env[63273]: DEBUG nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Instance network_info: |[{"id": "85f37247-c66a-4e8e-af09-6520b0d5a973", "address": "fa:16:3e:38:f5:8a", "network": {"id": "58774d83-9d0a-42ae-979d-a9c35214ca47", "bridge": "br-int", "label": "tempest-ServersNegativeTestJSON-2012108761-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "72f5eacfc9004694a5107a00edabbdd6", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "095fbf26-7367-4f9e-87c5-2965b64b0b0f", "external-id": "nsx-vlan-transportzone-777", "segmentation_id": 777, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap85f37247-c6", "ovs_interfaceid": "85f37247-c66a-4e8e-af09-6520b0d5a973", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1485.615575] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:38:f5:8a', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '095fbf26-7367-4f9e-87c5-2965b64b0b0f', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '85f37247-c66a-4e8e-af09-6520b0d5a973', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1485.623500] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Creating folder: Project (72f5eacfc9004694a5107a00edabbdd6). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1485.624110] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-166a63eb-affa-480f-a46f-d0334264b4ca {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1485.629160] env[63273]: DEBUG nova.network.neutron [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Successfully updated port: 4a713d74-61c8-4061-856f-83bfe2d702d5 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1485.637162] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Created folder: Project (72f5eacfc9004694a5107a00edabbdd6) in parent group-v986930. [ 1485.637378] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Creating folder: Instances. Parent ref: group-v987014. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1485.637627] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-088cc7da-db9d-43a7-9d5c-294b31237316 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1485.641343] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquiring lock "refresh_cache-d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1485.641553] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquired lock "refresh_cache-d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1485.641734] env[63273]: DEBUG nova.network.neutron [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1485.657019] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Created folder: Instances in parent group-v987014. [ 1485.657019] env[63273]: DEBUG oslo.service.loopingcall [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1485.657019] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1485.657019] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-2a716018-f75e-4482-adf7-b16e946d8b8c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1485.680929] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1485.680929] env[63273]: value = "task-5072132" [ 1485.680929] env[63273]: _type = "Task" [ 1485.680929] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1485.690029] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072132, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1485.691941] env[63273]: DEBUG nova.network.neutron [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1485.896268] env[63273]: DEBUG nova.network.neutron [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Updating instance_info_cache with network_info: [{"id": "4a713d74-61c8-4061-856f-83bfe2d702d5", "address": "fa:16:3e:5c:cf:87", "network": {"id": "085d2241-4def-4226-8111-64abe48f910b", "bridge": "br-int", "label": "tempest-ServerRescueTestJSON-1829342171-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": false}}], "meta": {"injected": false, "tenant_id": "d277c5d52dd44fbdab7560c6fb34cf31", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "26472e27-9835-4f87-ab7f-ca24dfee4e83", "external-id": "nsx-vlan-transportzone-335", "segmentation_id": 335, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4a713d74-61", "ovs_interfaceid": "4a713d74-61c8-4061-856f-83bfe2d702d5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1485.912270] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Releasing lock "refresh_cache-d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1485.912705] env[63273]: DEBUG nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Instance network_info: |[{"id": "4a713d74-61c8-4061-856f-83bfe2d702d5", "address": "fa:16:3e:5c:cf:87", "network": {"id": "085d2241-4def-4226-8111-64abe48f910b", "bridge": "br-int", "label": "tempest-ServerRescueTestJSON-1829342171-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": false}}], "meta": {"injected": false, "tenant_id": "d277c5d52dd44fbdab7560c6fb34cf31", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "26472e27-9835-4f87-ab7f-ca24dfee4e83", "external-id": "nsx-vlan-transportzone-335", "segmentation_id": 335, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4a713d74-61", "ovs_interfaceid": "4a713d74-61c8-4061-856f-83bfe2d702d5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1485.913457] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:5c:cf:87', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '26472e27-9835-4f87-ab7f-ca24dfee4e83', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '4a713d74-61c8-4061-856f-83bfe2d702d5', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1485.921770] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Creating folder: Project (d277c5d52dd44fbdab7560c6fb34cf31). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1485.922512] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-6d295d5e-48e2-4414-af54-f7b167c97019 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1485.934327] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Created folder: Project (d277c5d52dd44fbdab7560c6fb34cf31) in parent group-v986930. [ 1485.934581] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Creating folder: Instances. Parent ref: group-v987017. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1485.934822] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-686009bc-b2b7-4c60-8ce0-16bf147750fe {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1485.945182] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Created folder: Instances in parent group-v987017. [ 1485.945453] env[63273]: DEBUG oslo.service.loopingcall [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1485.945660] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1485.945875] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-60fc0568-3bc7-4558-9560-e066ed194ea8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1485.967055] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1485.967055] env[63273]: value = "task-5072135" [ 1485.967055] env[63273]: _type = "Task" [ 1485.967055] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1485.975458] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072135, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1486.192704] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072132, 'name': CreateVM_Task, 'duration_secs': 0.382151} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1486.192890] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1486.194018] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1486.194214] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1486.194787] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1486.195072] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-3c6d3b64-4b98-4857-a0b5-2808e8c50419 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1486.204845] env[63273]: DEBUG oslo_vmware.api [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Waiting for the task: (returnval){ [ 1486.204845] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]522fa0ee-f28d-7f43-799d-62f77365812b" [ 1486.204845] env[63273]: _type = "Task" [ 1486.204845] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1486.223846] env[63273]: DEBUG oslo_vmware.api [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]522fa0ee-f28d-7f43-799d-62f77365812b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1486.477730] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072135, 'name': CreateVM_Task, 'duration_secs': 0.397727} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1486.477909] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1486.478651] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1486.717065] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1486.717065] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1486.717289] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1486.717395] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1486.717707] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1486.717965] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-fdd54d9d-fe0e-4fbb-99b5-908cf2ca6043 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1486.723996] env[63273]: DEBUG oslo_vmware.api [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Waiting for the task: (returnval){ [ 1486.723996] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]527c5c88-7fec-1ecc-a379-d13670ef2fef" [ 1486.723996] env[63273]: _type = "Task" [ 1486.723996] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1486.731939] env[63273]: DEBUG oslo_vmware.api [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]527c5c88-7fec-1ecc-a379-d13670ef2fef, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1486.986942] env[63273]: DEBUG nova.compute.manager [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Received event network-changed-85f37247-c66a-4e8e-af09-6520b0d5a973 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1486.987209] env[63273]: DEBUG nova.compute.manager [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Refreshing instance network info cache due to event network-changed-85f37247-c66a-4e8e-af09-6520b0d5a973. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1486.987325] env[63273]: DEBUG oslo_concurrency.lockutils [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] Acquiring lock "refresh_cache-efe849d5-9df6-4813-a23b-c805e7eb2456" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1486.987469] env[63273]: DEBUG oslo_concurrency.lockutils [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] Acquired lock "refresh_cache-efe849d5-9df6-4813-a23b-c805e7eb2456" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1486.987632] env[63273]: DEBUG nova.network.neutron [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Refreshing network info cache for port 85f37247-c66a-4e8e-af09-6520b0d5a973 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1487.235684] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1487.235986] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1487.236232] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1487.342260] env[63273]: DEBUG nova.network.neutron [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Updated VIF entry in instance network info cache for port 85f37247-c66a-4e8e-af09-6520b0d5a973. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1487.342526] env[63273]: DEBUG nova.network.neutron [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Updating instance_info_cache with network_info: [{"id": "85f37247-c66a-4e8e-af09-6520b0d5a973", "address": "fa:16:3e:38:f5:8a", "network": {"id": "58774d83-9d0a-42ae-979d-a9c35214ca47", "bridge": "br-int", "label": "tempest-ServersNegativeTestJSON-2012108761-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "72f5eacfc9004694a5107a00edabbdd6", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "095fbf26-7367-4f9e-87c5-2965b64b0b0f", "external-id": "nsx-vlan-transportzone-777", "segmentation_id": 777, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap85f37247-c6", "ovs_interfaceid": "85f37247-c66a-4e8e-af09-6520b0d5a973", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1487.352605] env[63273]: DEBUG oslo_concurrency.lockutils [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] Releasing lock "refresh_cache-efe849d5-9df6-4813-a23b-c805e7eb2456" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1487.352845] env[63273]: DEBUG nova.compute.manager [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Received event network-vif-plugged-4a713d74-61c8-4061-856f-83bfe2d702d5 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1487.353068] env[63273]: DEBUG oslo_concurrency.lockutils [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] Acquiring lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1487.353326] env[63273]: DEBUG oslo_concurrency.lockutils [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] Lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1487.353443] env[63273]: DEBUG oslo_concurrency.lockutils [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] Lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1487.353606] env[63273]: DEBUG nova.compute.manager [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] No waiting events found dispatching network-vif-plugged-4a713d74-61c8-4061-856f-83bfe2d702d5 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1487.353797] env[63273]: WARNING nova.compute.manager [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Received unexpected event network-vif-plugged-4a713d74-61c8-4061-856f-83bfe2d702d5 for instance with vm_state building and task_state spawning. [ 1487.353977] env[63273]: DEBUG nova.compute.manager [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Received event network-changed-4a713d74-61c8-4061-856f-83bfe2d702d5 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1487.354150] env[63273]: DEBUG nova.compute.manager [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Refreshing instance network info cache due to event network-changed-4a713d74-61c8-4061-856f-83bfe2d702d5. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1487.354329] env[63273]: DEBUG oslo_concurrency.lockutils [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] Acquiring lock "refresh_cache-d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1487.354465] env[63273]: DEBUG oslo_concurrency.lockutils [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] Acquired lock "refresh_cache-d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1487.354619] env[63273]: DEBUG nova.network.neutron [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Refreshing network info cache for port 4a713d74-61c8-4061-856f-83bfe2d702d5 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1487.645524] env[63273]: DEBUG oslo_concurrency.lockutils [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "1da174da-b4e6-437c-a538-53fc78b4282f" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1487.740798] env[63273]: DEBUG nova.network.neutron [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Updated VIF entry in instance network info cache for port 4a713d74-61c8-4061-856f-83bfe2d702d5. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1487.741379] env[63273]: DEBUG nova.network.neutron [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Updating instance_info_cache with network_info: [{"id": "4a713d74-61c8-4061-856f-83bfe2d702d5", "address": "fa:16:3e:5c:cf:87", "network": {"id": "085d2241-4def-4226-8111-64abe48f910b", "bridge": "br-int", "label": "tempest-ServerRescueTestJSON-1829342171-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": false}}], "meta": {"injected": false, "tenant_id": "d277c5d52dd44fbdab7560c6fb34cf31", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "26472e27-9835-4f87-ab7f-ca24dfee4e83", "external-id": "nsx-vlan-transportzone-335", "segmentation_id": 335, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4a713d74-61", "ovs_interfaceid": "4a713d74-61c8-4061-856f-83bfe2d702d5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1487.752335] env[63273]: DEBUG oslo_concurrency.lockutils [req-b1d6bb9a-c7f5-4935-a496-d53ed8ecc4b9 req-fc63c7be-c55b-422e-b959-0f4821671408 service nova] Releasing lock "refresh_cache-d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1499.206244] env[63273]: DEBUG oslo_concurrency.lockutils [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1529.891459] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1530.344165] env[63273]: WARNING oslo_vmware.rw_handles [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1530.344165] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1530.344165] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1530.344165] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1530.344165] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1530.344165] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1530.344165] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1530.344165] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1530.344165] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1530.344165] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1530.344165] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1530.344165] env[63273]: ERROR oslo_vmware.rw_handles [ 1530.345347] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/c873d914-7f34-4348-b735-c9e3fa8103f3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1530.346749] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1530.346998] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Copying Virtual Disk [datastore1] vmware_temp/c873d914-7f34-4348-b735-c9e3fa8103f3/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/c873d914-7f34-4348-b735-c9e3fa8103f3/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1530.347319] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-f67ae5e6-eca1-4e9a-9dd8-f7d8eda41dd8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1530.355749] env[63273]: DEBUG oslo_vmware.api [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Waiting for the task: (returnval){ [ 1530.355749] env[63273]: value = "task-5072136" [ 1530.355749] env[63273]: _type = "Task" [ 1530.355749] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1530.364686] env[63273]: DEBUG oslo_vmware.api [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Task: {'id': task-5072136, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1530.866570] env[63273]: DEBUG oslo_vmware.exceptions [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1530.866913] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1530.867506] env[63273]: ERROR nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1530.867506] env[63273]: Faults: ['InvalidArgument'] [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Traceback (most recent call last): [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] yield resources [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] self.driver.spawn(context, instance, image_meta, [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] self._fetch_image_if_missing(context, vi) [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] image_cache(vi, tmp_image_ds_loc) [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] vm_util.copy_virtual_disk( [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] session._wait_for_task(vmdk_copy_task) [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] return self.wait_for_task(task_ref) [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] return evt.wait() [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] result = hub.switch() [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] return self.greenlet.switch() [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] self.f(*self.args, **self.kw) [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] raise exceptions.translate_fault(task_info.error) [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Faults: ['InvalidArgument'] [ 1530.867506] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] [ 1530.868546] env[63273]: INFO nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Terminating instance [ 1530.869499] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1530.869711] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1530.869957] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-976f8f53-a180-49f8-8920-d1194ddcb319 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1530.872379] env[63273]: DEBUG nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1530.872572] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1530.873303] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93f53761-e00e-4281-bbd6-0117e1202095 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1530.879909] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1530.880150] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-775d4612-c69d-4435-a424-6fd903e5b299 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1530.882354] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1530.882529] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1530.883497] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f5a7e412-19ec-4e04-9fdb-e7e10d471268 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1530.888443] env[63273]: DEBUG oslo_vmware.api [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Waiting for the task: (returnval){ [ 1530.888443] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5268475e-2c16-012e-526b-48ef1170e9b6" [ 1530.888443] env[63273]: _type = "Task" [ 1530.888443] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1530.891425] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1530.897149] env[63273]: DEBUG oslo_vmware.api [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5268475e-2c16-012e-526b-48ef1170e9b6, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1530.903571] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1530.903873] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1530.904134] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1530.904363] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1530.905721] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5df8ecc-fc71-40bc-bfef-f5e600bba116 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1530.916308] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5863f08a-1a69-4ce7-bfba-60b1e9f72065 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1530.930917] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32817174-6600-4976-a29e-2cfcc5224f0e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1530.938084] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea142403-3133-4e09-a350-7c00d31767d8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1530.968348] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180513MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1530.968576] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1530.968685] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1530.971711] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1530.971920] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1530.972117] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Deleting the datastore file [datastore1] fefdf558-7a73-4bae-b57c-b86963189ddb {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1530.972593] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-2755c0a4-f471-479e-82b1-857744408bfe {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1530.979228] env[63273]: DEBUG oslo_vmware.api [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Waiting for the task: (returnval){ [ 1530.979228] env[63273]: value = "task-5072138" [ 1530.979228] env[63273]: _type = "Task" [ 1530.979228] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1530.987835] env[63273]: DEBUG oslo_vmware.api [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Task: {'id': task-5072138, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1531.043612] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fefdf558-7a73-4bae-b57c-b86963189ddb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1531.043816] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance a081322d-4636-4bfc-90f6-bd0c617a09e8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1531.043978] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1531.044152] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1531.044295] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1531.044419] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1531.044558] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fa31b240-8bad-48ba-8339-155dc6acb265 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1531.044701] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1da174da-b4e6-437c-a538-53fc78b4282f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1531.044850] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance efe849d5-9df6-4813-a23b-c805e7eb2456 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1531.044985] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1531.057182] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 61bd6d66-590f-4512-afbf-9abf4a308749 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1531.069515] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1531.081398] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance dd885558-e9a6-490e-96fa-a4d82db45c5c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1531.081638] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1531.081810] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '71', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '8', 'num_os_type_None': '10', 'num_proj_ad0ab37da3b64969894f8ab378f35ff7': '1', 'io_workload': '10', 'num_proj_bc9a82a949b84d2db6b4602db73e0d2e': '1', 'num_proj_71b2d87fd7ff416db331d00f7375e59b': '2', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1', 'num_proj_6a6c31a7f8364be284dfcc2d167ed810': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_task_spawning': '2', 'num_proj_72f5eacfc9004694a5107a00edabbdd6': '1', 'num_proj_d277c5d52dd44fbdab7560c6fb34cf31': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1531.264527] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1129da3e-0e6e-493d-ae1c-e824211bd6c1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.272430] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90639ede-6715-471c-91ba-df5e109b7df7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.303966] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f4bcce7-d620-4db4-81cc-35a31d65ca04 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.312364] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8b2b083-a26e-4f1c-8bb2-90ed5a8960d3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.326218] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1531.335185] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1531.349265] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1531.349451] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.381s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1531.399336] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1531.399598] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Creating directory with path [datastore1] vmware_temp/199a9513-c9b5-4c77-b0e5-c63f6a47aabe/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1531.399840] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-9e3a9bab-d779-4a50-94d9-32e4ad6a91e3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.412638] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Created directory with path [datastore1] vmware_temp/199a9513-c9b5-4c77-b0e5-c63f6a47aabe/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1531.412855] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Fetch image to [datastore1] vmware_temp/199a9513-c9b5-4c77-b0e5-c63f6a47aabe/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1531.413028] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/199a9513-c9b5-4c77-b0e5-c63f6a47aabe/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1531.414071] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c29e80e-a4f3-4d6f-b620-457b917122b7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.421618] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b809614-b706-4960-8ff7-e758e1d18cc7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.431811] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2250155-cca3-461b-b90e-4e531ccd0e68 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.463733] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95d394f5-b0c2-471a-b63b-a0a95c3c6278 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.470296] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-6d6d5cb1-a7f1-423e-8bf0-bd8a11562084 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.489381] env[63273]: DEBUG oslo_vmware.api [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Task: {'id': task-5072138, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.079549} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1531.489653] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1531.489836] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1531.490024] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1531.490255] env[63273]: INFO nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1531.492471] env[63273]: DEBUG nova.compute.claims [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1531.492751] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1531.492894] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1531.497282] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1531.552564] env[63273]: DEBUG oslo_vmware.rw_handles [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/199a9513-c9b5-4c77-b0e5-c63f6a47aabe/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1531.616640] env[63273]: DEBUG oslo_vmware.rw_handles [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1531.616796] env[63273]: DEBUG oslo_vmware.rw_handles [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/199a9513-c9b5-4c77-b0e5-c63f6a47aabe/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1531.756710] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d522c0f6-2288-49d4-8417-05a533d30465 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.764626] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1212abe8-ddd0-4927-92b2-5089edd6d65f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.793576] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f05e63f0-f16b-4687-8808-19bfc7fa66d8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.800582] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a615890-d672-4237-b8e0-988c4cd67c9e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1531.814532] env[63273]: DEBUG nova.compute.provider_tree [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1531.822989] env[63273]: DEBUG nova.scheduler.client.report [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1531.836580] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.344s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1531.837110] env[63273]: ERROR nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1531.837110] env[63273]: Faults: ['InvalidArgument'] [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Traceback (most recent call last): [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] self.driver.spawn(context, instance, image_meta, [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] self._fetch_image_if_missing(context, vi) [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] image_cache(vi, tmp_image_ds_loc) [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] vm_util.copy_virtual_disk( [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] session._wait_for_task(vmdk_copy_task) [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] return self.wait_for_task(task_ref) [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] return evt.wait() [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] result = hub.switch() [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] return self.greenlet.switch() [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] self.f(*self.args, **self.kw) [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] raise exceptions.translate_fault(task_info.error) [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Faults: ['InvalidArgument'] [ 1531.837110] env[63273]: ERROR nova.compute.manager [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] [ 1531.837931] env[63273]: DEBUG nova.compute.utils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1531.839485] env[63273]: DEBUG nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Build of instance fefdf558-7a73-4bae-b57c-b86963189ddb was re-scheduled: A specified parameter was not correct: fileType [ 1531.839485] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1531.839877] env[63273]: DEBUG nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1531.840059] env[63273]: DEBUG nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1531.840232] env[63273]: DEBUG nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1531.840396] env[63273]: DEBUG nova.network.neutron [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1532.158430] env[63273]: DEBUG nova.network.neutron [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1532.173511] env[63273]: INFO nova.compute.manager [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Took 0.33 seconds to deallocate network for instance. [ 1532.288439] env[63273]: INFO nova.scheduler.client.report [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Deleted allocations for instance fefdf558-7a73-4bae-b57c-b86963189ddb [ 1532.321075] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ba5671c9-b2e7-4deb-85b1-4e20f556dbcb tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Lock "fefdf558-7a73-4bae-b57c-b86963189ddb" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 567.430s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1532.322688] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Lock "fefdf558-7a73-4bae-b57c-b86963189ddb" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 370.386s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1532.322688] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Acquiring lock "fefdf558-7a73-4bae-b57c-b86963189ddb-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1532.322884] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Lock "fefdf558-7a73-4bae-b57c-b86963189ddb-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1532.322939] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Lock "fefdf558-7a73-4bae-b57c-b86963189ddb-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1532.325358] env[63273]: INFO nova.compute.manager [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Terminating instance [ 1532.327118] env[63273]: DEBUG nova.compute.manager [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1532.327238] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1532.327721] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3bd4e3d1-1441-4782-9933-fd06175bf935 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1532.342594] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa3d251d-7d41-45c8-91f5-09ad31523777 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1532.355282] env[63273]: DEBUG nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1532.382283] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fefdf558-7a73-4bae-b57c-b86963189ddb could not be found. [ 1532.382283] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1532.382492] env[63273]: INFO nova.compute.manager [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Took 0.06 seconds to destroy the instance on the hypervisor. [ 1532.383100] env[63273]: DEBUG oslo.service.loopingcall [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1532.383100] env[63273]: DEBUG nova.compute.manager [-] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1532.383100] env[63273]: DEBUG nova.network.neutron [-] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1532.409046] env[63273]: DEBUG nova.network.neutron [-] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1532.411608] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1532.411876] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1532.413416] env[63273]: INFO nova.compute.claims [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1532.417640] env[63273]: INFO nova.compute.manager [-] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] Took 0.03 seconds to deallocate network for instance. [ 1532.553271] env[63273]: DEBUG oslo_concurrency.lockutils [None req-9ff3c951-b808-4977-895b-74a2cf2e8e9a tempest-ImagesOneServerTestJSON-1469685948 tempest-ImagesOneServerTestJSON-1469685948-project-member] Lock "fefdf558-7a73-4bae-b57c-b86963189ddb" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.231s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1532.554153] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "fefdf558-7a73-4bae-b57c-b86963189ddb" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 368.329s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1532.554342] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fefdf558-7a73-4bae-b57c-b86963189ddb] During sync_power_state the instance has a pending task (deleting). Skip. [ 1532.554516] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "fefdf558-7a73-4bae-b57c-b86963189ddb" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1532.657509] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-203e24ac-3eba-4e2a-a500-e2d6bc4b50f6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1532.665778] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-136c4c49-713a-4599-9c33-c20c9d7dd524 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1532.698089] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c988602b-bdb2-48e1-8cd3-c4d8609d0179 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1532.706794] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-deeda890-ccc2-497d-9045-3fe58e3b70ea {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1532.720920] env[63273]: DEBUG nova.compute.provider_tree [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1532.731145] env[63273]: DEBUG nova.scheduler.client.report [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1532.745629] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.333s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1532.745885] env[63273]: DEBUG nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1532.781747] env[63273]: DEBUG nova.compute.utils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1532.783081] env[63273]: DEBUG nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1532.783245] env[63273]: DEBUG nova.network.neutron [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1532.792029] env[63273]: DEBUG nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1532.858286] env[63273]: DEBUG nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1532.861966] env[63273]: DEBUG nova.policy [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fe186bc3d5b043b691d0927cb8be722a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a5420f4321c84264957a02c02f47649f', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1532.888605] env[63273]: DEBUG nova.virt.hardware [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1532.888853] env[63273]: DEBUG nova.virt.hardware [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1532.889062] env[63273]: DEBUG nova.virt.hardware [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1532.889305] env[63273]: DEBUG nova.virt.hardware [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1532.889458] env[63273]: DEBUG nova.virt.hardware [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1532.889608] env[63273]: DEBUG nova.virt.hardware [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1532.889823] env[63273]: DEBUG nova.virt.hardware [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1532.889984] env[63273]: DEBUG nova.virt.hardware [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1532.890165] env[63273]: DEBUG nova.virt.hardware [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1532.890332] env[63273]: DEBUG nova.virt.hardware [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1532.890536] env[63273]: DEBUG nova.virt.hardware [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1532.891422] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd38ac26-92f4-4016-96ce-19649772eab5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1532.899588] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3a6e5e07-a06f-42df-b292-e53c317cbe0f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1533.172986] env[63273]: DEBUG nova.network.neutron [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Successfully created port: 59420cbb-1469-4d60-a977-c6bf9cd2fdd2 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1533.350667] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1533.891151] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1534.224891] env[63273]: DEBUG nova.compute.manager [req-9966e7db-40a1-4475-a96d-3d2dee2e7a2e req-1a2c5df5-bb44-45e7-a9db-bdc81a38f3aa service nova] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Received event network-vif-plugged-59420cbb-1469-4d60-a977-c6bf9cd2fdd2 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1534.225175] env[63273]: DEBUG oslo_concurrency.lockutils [req-9966e7db-40a1-4475-a96d-3d2dee2e7a2e req-1a2c5df5-bb44-45e7-a9db-bdc81a38f3aa service nova] Acquiring lock "61bd6d66-590f-4512-afbf-9abf4a308749-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1534.225509] env[63273]: DEBUG oslo_concurrency.lockutils [req-9966e7db-40a1-4475-a96d-3d2dee2e7a2e req-1a2c5df5-bb44-45e7-a9db-bdc81a38f3aa service nova] Lock "61bd6d66-590f-4512-afbf-9abf4a308749-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1534.225509] env[63273]: DEBUG oslo_concurrency.lockutils [req-9966e7db-40a1-4475-a96d-3d2dee2e7a2e req-1a2c5df5-bb44-45e7-a9db-bdc81a38f3aa service nova] Lock "61bd6d66-590f-4512-afbf-9abf4a308749-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1534.225638] env[63273]: DEBUG nova.compute.manager [req-9966e7db-40a1-4475-a96d-3d2dee2e7a2e req-1a2c5df5-bb44-45e7-a9db-bdc81a38f3aa service nova] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] No waiting events found dispatching network-vif-plugged-59420cbb-1469-4d60-a977-c6bf9cd2fdd2 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1534.225800] env[63273]: WARNING nova.compute.manager [req-9966e7db-40a1-4475-a96d-3d2dee2e7a2e req-1a2c5df5-bb44-45e7-a9db-bdc81a38f3aa service nova] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Received unexpected event network-vif-plugged-59420cbb-1469-4d60-a977-c6bf9cd2fdd2 for instance with vm_state building and task_state spawning. [ 1534.279855] env[63273]: DEBUG nova.network.neutron [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Successfully updated port: 59420cbb-1469-4d60-a977-c6bf9cd2fdd2 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1534.294883] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquiring lock "refresh_cache-61bd6d66-590f-4512-afbf-9abf4a308749" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1534.294883] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquired lock "refresh_cache-61bd6d66-590f-4512-afbf-9abf4a308749" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1534.294883] env[63273]: DEBUG nova.network.neutron [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1534.338390] env[63273]: DEBUG nova.network.neutron [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1534.555998] env[63273]: DEBUG nova.network.neutron [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Updating instance_info_cache with network_info: [{"id": "59420cbb-1469-4d60-a977-c6bf9cd2fdd2", "address": "fa:16:3e:dc:c8:e8", "network": {"id": "03c650e2-20c2-4e93-9be8-3d9f4dbebe7f", "bridge": "br-int", "label": "tempest-ImagesTestJSON-1918952271-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "a5420f4321c84264957a02c02f47649f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e365f3b9-706b-4fa2-8f95-ae51b35ab011", "external-id": "nsx-vlan-transportzone-154", "segmentation_id": 154, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap59420cbb-14", "ovs_interfaceid": "59420cbb-1469-4d60-a977-c6bf9cd2fdd2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1534.568323] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Releasing lock "refresh_cache-61bd6d66-590f-4512-afbf-9abf4a308749" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1534.568630] env[63273]: DEBUG nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Instance network_info: |[{"id": "59420cbb-1469-4d60-a977-c6bf9cd2fdd2", "address": "fa:16:3e:dc:c8:e8", "network": {"id": "03c650e2-20c2-4e93-9be8-3d9f4dbebe7f", "bridge": "br-int", "label": "tempest-ImagesTestJSON-1918952271-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "a5420f4321c84264957a02c02f47649f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e365f3b9-706b-4fa2-8f95-ae51b35ab011", "external-id": "nsx-vlan-transportzone-154", "segmentation_id": 154, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap59420cbb-14", "ovs_interfaceid": "59420cbb-1469-4d60-a977-c6bf9cd2fdd2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1534.569062] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:dc:c8:e8', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'e365f3b9-706b-4fa2-8f95-ae51b35ab011', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '59420cbb-1469-4d60-a977-c6bf9cd2fdd2', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1534.576635] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Creating folder: Project (a5420f4321c84264957a02c02f47649f). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1534.577236] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d9b74059-2ebe-46d3-899d-d0a37c4c5e10 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1534.589295] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Created folder: Project (a5420f4321c84264957a02c02f47649f) in parent group-v986930. [ 1534.589295] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Creating folder: Instances. Parent ref: group-v987020. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1534.589295] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-9efc054b-7c41-4051-accc-036f9faf0549 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1534.599994] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Created folder: Instances in parent group-v987020. [ 1534.599994] env[63273]: DEBUG oslo.service.loopingcall [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1534.600207] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1534.600405] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-fc8da452-1d95-43bf-818f-00a1088ac200 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1534.621349] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1534.621349] env[63273]: value = "task-5072141" [ 1534.621349] env[63273]: _type = "Task" [ 1534.621349] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1534.633095] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072141, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1535.131505] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072141, 'name': CreateVM_Task, 'duration_secs': 0.317816} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1535.131826] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1535.132376] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1535.132545] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1535.132873] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1535.133169] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-7a95dad1-ac26-4ae8-b20f-7229c1b7cdc7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1535.137743] env[63273]: DEBUG oslo_vmware.api [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Waiting for the task: (returnval){ [ 1535.137743] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]529d1d5d-4d70-a67d-314b-d633717e6c0a" [ 1535.137743] env[63273]: _type = "Task" [ 1535.137743] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1535.145678] env[63273]: DEBUG oslo_vmware.api [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]529d1d5d-4d70-a67d-314b-d633717e6c0a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1535.648682] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1535.648682] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1535.648682] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1535.893083] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1535.893083] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1535.893083] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1535.914414] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1535.914555] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1535.914645] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1535.914765] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1535.914892] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1535.915025] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1535.915158] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1535.915278] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1535.915397] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1535.915513] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1535.915632] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1535.916315] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1536.276472] env[63273]: DEBUG nova.compute.manager [req-66a72d79-352f-425e-890e-63b49a573460 req-6f12a54d-771e-4c11-9dc0-7ff608525bcc service nova] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Received event network-changed-59420cbb-1469-4d60-a977-c6bf9cd2fdd2 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1536.276679] env[63273]: DEBUG nova.compute.manager [req-66a72d79-352f-425e-890e-63b49a573460 req-6f12a54d-771e-4c11-9dc0-7ff608525bcc service nova] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Refreshing instance network info cache due to event network-changed-59420cbb-1469-4d60-a977-c6bf9cd2fdd2. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1536.276891] env[63273]: DEBUG oslo_concurrency.lockutils [req-66a72d79-352f-425e-890e-63b49a573460 req-6f12a54d-771e-4c11-9dc0-7ff608525bcc service nova] Acquiring lock "refresh_cache-61bd6d66-590f-4512-afbf-9abf4a308749" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1536.277058] env[63273]: DEBUG oslo_concurrency.lockutils [req-66a72d79-352f-425e-890e-63b49a573460 req-6f12a54d-771e-4c11-9dc0-7ff608525bcc service nova] Acquired lock "refresh_cache-61bd6d66-590f-4512-afbf-9abf4a308749" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1536.277231] env[63273]: DEBUG nova.network.neutron [req-66a72d79-352f-425e-890e-63b49a573460 req-6f12a54d-771e-4c11-9dc0-7ff608525bcc service nova] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Refreshing network info cache for port 59420cbb-1469-4d60-a977-c6bf9cd2fdd2 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1536.634478] env[63273]: DEBUG nova.network.neutron [req-66a72d79-352f-425e-890e-63b49a573460 req-6f12a54d-771e-4c11-9dc0-7ff608525bcc service nova] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Updated VIF entry in instance network info cache for port 59420cbb-1469-4d60-a977-c6bf9cd2fdd2. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1536.635078] env[63273]: DEBUG nova.network.neutron [req-66a72d79-352f-425e-890e-63b49a573460 req-6f12a54d-771e-4c11-9dc0-7ff608525bcc service nova] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Updating instance_info_cache with network_info: [{"id": "59420cbb-1469-4d60-a977-c6bf9cd2fdd2", "address": "fa:16:3e:dc:c8:e8", "network": {"id": "03c650e2-20c2-4e93-9be8-3d9f4dbebe7f", "bridge": "br-int", "label": "tempest-ImagesTestJSON-1918952271-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "a5420f4321c84264957a02c02f47649f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e365f3b9-706b-4fa2-8f95-ae51b35ab011", "external-id": "nsx-vlan-transportzone-154", "segmentation_id": 154, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap59420cbb-14", "ovs_interfaceid": "59420cbb-1469-4d60-a977-c6bf9cd2fdd2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1536.645597] env[63273]: DEBUG oslo_concurrency.lockutils [req-66a72d79-352f-425e-890e-63b49a573460 req-6f12a54d-771e-4c11-9dc0-7ff608525bcc service nova] Releasing lock "refresh_cache-61bd6d66-590f-4512-afbf-9abf4a308749" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1537.891115] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1537.891455] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1537.891496] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1540.887132] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1542.403961] env[63273]: DEBUG oslo_concurrency.lockutils [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquiring lock "efe849d5-9df6-4813-a23b-c805e7eb2456" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1553.270341] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquiring lock "92426c26-52fa-4978-a382-3684ec018bc5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1553.270749] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Lock "92426c26-52fa-4978-a382-3684ec018bc5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1560.912773] env[63273]: DEBUG oslo_concurrency.lockutils [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquiring lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1578.569170] env[63273]: WARNING oslo_vmware.rw_handles [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1578.569170] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1578.569170] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1578.569170] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1578.569170] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1578.569170] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1578.569170] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1578.569170] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1578.569170] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1578.569170] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1578.569170] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1578.569170] env[63273]: ERROR oslo_vmware.rw_handles [ 1578.569935] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/199a9513-c9b5-4c77-b0e5-c63f6a47aabe/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1578.571961] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1578.572323] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Copying Virtual Disk [datastore1] vmware_temp/199a9513-c9b5-4c77-b0e5-c63f6a47aabe/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/199a9513-c9b5-4c77-b0e5-c63f6a47aabe/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1578.572652] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-6a98ba63-8866-453d-a30c-4c879ad8b90b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1578.581469] env[63273]: DEBUG oslo_vmware.api [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Waiting for the task: (returnval){ [ 1578.581469] env[63273]: value = "task-5072142" [ 1578.581469] env[63273]: _type = "Task" [ 1578.581469] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1578.590295] env[63273]: DEBUG oslo_vmware.api [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Task: {'id': task-5072142, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1579.093380] env[63273]: DEBUG oslo_vmware.exceptions [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1579.093829] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1579.094447] env[63273]: ERROR nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1579.094447] env[63273]: Faults: ['InvalidArgument'] [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Traceback (most recent call last): [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] yield resources [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] self.driver.spawn(context, instance, image_meta, [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] self._fetch_image_if_missing(context, vi) [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] image_cache(vi, tmp_image_ds_loc) [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] vm_util.copy_virtual_disk( [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] session._wait_for_task(vmdk_copy_task) [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] return self.wait_for_task(task_ref) [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] return evt.wait() [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] result = hub.switch() [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] return self.greenlet.switch() [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] self.f(*self.args, **self.kw) [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] raise exceptions.translate_fault(task_info.error) [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Faults: ['InvalidArgument'] [ 1579.094447] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] [ 1579.095829] env[63273]: INFO nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Terminating instance [ 1579.097240] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1579.097450] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1579.098110] env[63273]: DEBUG nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1579.098313] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1579.098551] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-c9112efc-f1b4-486c-974a-d694b5d35187 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1579.101215] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-644e1f70-5882-461d-887d-eef83b7c77f9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1579.109945] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1579.109945] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-64eec3f8-fb43-4bb0-8006-69f1d4fca4a2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1579.111444] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1579.111612] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1579.112295] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-1a4f11cf-89e7-4211-969b-071fcc82c42e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1579.119870] env[63273]: DEBUG oslo_vmware.api [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for the task: (returnval){ [ 1579.119870] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]521d5472-3fd3-bb31-559d-2a8d89a93ea8" [ 1579.119870] env[63273]: _type = "Task" [ 1579.119870] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1579.128370] env[63273]: DEBUG oslo_vmware.api [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]521d5472-3fd3-bb31-559d-2a8d89a93ea8, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1579.185930] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1579.186194] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1579.186386] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Deleting the datastore file [datastore1] a081322d-4636-4bfc-90f6-bd0c617a09e8 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1579.186808] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-7f9b421f-c128-406e-a426-7b54124598db {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1579.194879] env[63273]: DEBUG oslo_vmware.api [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Waiting for the task: (returnval){ [ 1579.194879] env[63273]: value = "task-5072144" [ 1579.194879] env[63273]: _type = "Task" [ 1579.194879] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1579.202523] env[63273]: DEBUG oslo_vmware.api [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Task: {'id': task-5072144, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1579.630962] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1579.631425] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Creating directory with path [datastore1] vmware_temp/23dcf226-7a8c-4e8b-98c1-bf02296bcede/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1579.631507] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-b7d4445e-b774-429d-9e00-9be3d1647695 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1579.645047] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Created directory with path [datastore1] vmware_temp/23dcf226-7a8c-4e8b-98c1-bf02296bcede/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1579.645218] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Fetch image to [datastore1] vmware_temp/23dcf226-7a8c-4e8b-98c1-bf02296bcede/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1579.645369] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/23dcf226-7a8c-4e8b-98c1-bf02296bcede/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1579.646160] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45cb3244-8efa-451f-b8a4-bfaea7871c5f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1579.654230] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c51b5009-d125-4fd5-9ec6-682d715894dd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1579.666268] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-23bce325-aa41-4231-a272-48d96bfa682a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1579.701925] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2078984-4fd0-4659-a14e-e1d24c9949a8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1579.712524] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-3b8f460d-6ae8-41ec-be32-98101a36d077 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1579.714656] env[63273]: DEBUG oslo_vmware.api [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Task: {'id': task-5072144, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.084051} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1579.714914] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1579.715116] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1579.715387] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1579.715543] env[63273]: INFO nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1579.717898] env[63273]: DEBUG nova.compute.claims [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1579.718091] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1579.718314] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1579.764254] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1579.823814] env[63273]: DEBUG oslo_vmware.rw_handles [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/23dcf226-7a8c-4e8b-98c1-bf02296bcede/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1579.887418] env[63273]: DEBUG oslo_vmware.rw_handles [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1579.887624] env[63273]: DEBUG oslo_vmware.rw_handles [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/23dcf226-7a8c-4e8b-98c1-bf02296bcede/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1580.026048] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e64b1d5a-2f62-40aa-84a9-f6ed2b148fc7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1580.036764] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29f86226-6454-42ac-8bff-49b8b0b9edda {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1580.068495] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e339f8d-244c-482f-ad1b-876cd979e01a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1580.077645] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a9a0770-db4f-462b-b5a3-d6909046db08 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1580.092478] env[63273]: DEBUG nova.compute.provider_tree [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1580.103584] env[63273]: DEBUG nova.scheduler.client.report [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1580.119684] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.401s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1580.120270] env[63273]: ERROR nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1580.120270] env[63273]: Faults: ['InvalidArgument'] [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Traceback (most recent call last): [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] self.driver.spawn(context, instance, image_meta, [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] self._fetch_image_if_missing(context, vi) [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] image_cache(vi, tmp_image_ds_loc) [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] vm_util.copy_virtual_disk( [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] session._wait_for_task(vmdk_copy_task) [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] return self.wait_for_task(task_ref) [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] return evt.wait() [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] result = hub.switch() [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] return self.greenlet.switch() [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] self.f(*self.args, **self.kw) [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] raise exceptions.translate_fault(task_info.error) [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Faults: ['InvalidArgument'] [ 1580.120270] env[63273]: ERROR nova.compute.manager [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] [ 1580.121882] env[63273]: DEBUG nova.compute.utils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1580.123136] env[63273]: DEBUG nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Build of instance a081322d-4636-4bfc-90f6-bd0c617a09e8 was re-scheduled: A specified parameter was not correct: fileType [ 1580.123136] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1580.123596] env[63273]: DEBUG nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1580.123771] env[63273]: DEBUG nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1580.124052] env[63273]: DEBUG nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1580.124122] env[63273]: DEBUG nova.network.neutron [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1580.579826] env[63273]: DEBUG nova.network.neutron [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1580.596349] env[63273]: INFO nova.compute.manager [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Took 0.47 seconds to deallocate network for instance. [ 1580.721903] env[63273]: INFO nova.scheduler.client.report [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Deleted allocations for instance a081322d-4636-4bfc-90f6-bd0c617a09e8 [ 1580.750027] env[63273]: DEBUG oslo_concurrency.lockutils [None req-17859691-2fb3-435a-8b49-379114919499 tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Lock "a081322d-4636-4bfc-90f6-bd0c617a09e8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 579.206s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1580.750027] env[63273]: DEBUG oslo_concurrency.lockutils [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Lock "a081322d-4636-4bfc-90f6-bd0c617a09e8" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 383.374s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1580.750027] env[63273]: DEBUG oslo_concurrency.lockutils [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Acquiring lock "a081322d-4636-4bfc-90f6-bd0c617a09e8-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1580.750027] env[63273]: DEBUG oslo_concurrency.lockutils [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Lock "a081322d-4636-4bfc-90f6-bd0c617a09e8-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1580.750027] env[63273]: DEBUG oslo_concurrency.lockutils [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Lock "a081322d-4636-4bfc-90f6-bd0c617a09e8-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1580.752294] env[63273]: INFO nova.compute.manager [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Terminating instance [ 1580.754398] env[63273]: DEBUG nova.compute.manager [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1580.754770] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1580.755450] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-09644165-8de2-4df2-9bf2-20ffbf655340 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1580.766799] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03d31766-29f6-49da-981a-122b23c54c54 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1580.780699] env[63273]: DEBUG nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1580.805499] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a081322d-4636-4bfc-90f6-bd0c617a09e8 could not be found. [ 1580.805679] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1580.805867] env[63273]: INFO nova.compute.manager [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1580.806141] env[63273]: DEBUG oslo.service.loopingcall [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1580.806448] env[63273]: DEBUG nova.compute.manager [-] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1580.806547] env[63273]: DEBUG nova.network.neutron [-] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1580.836694] env[63273]: DEBUG nova.network.neutron [-] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1580.847627] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1580.848263] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1580.850463] env[63273]: INFO nova.compute.claims [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1580.858020] env[63273]: INFO nova.compute.manager [-] [instance: a081322d-4636-4bfc-90f6-bd0c617a09e8] Took 0.05 seconds to deallocate network for instance. [ 1580.975838] env[63273]: DEBUG oslo_concurrency.lockutils [None req-730bbeaf-7f63-4ad4-9c23-a68a1618c38e tempest-ServersV294TestFqdnHostnames-558291287 tempest-ServersV294TestFqdnHostnames-558291287-project-member] Lock "a081322d-4636-4bfc-90f6-bd0c617a09e8" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.227s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1581.101362] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4bec14f0-c83d-4806-a772-89f7f5a245b0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1581.109511] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bebaff48-3163-4612-8a21-cc6c38f72106 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1581.140996] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-efc3bc8a-cbcc-4a3d-a739-c234cdf4fb93 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1581.149146] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-283d6e59-c32d-4953-bf0c-08e6a569a350 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1581.163159] env[63273]: DEBUG nova.compute.provider_tree [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1581.177103] env[63273]: DEBUG nova.scheduler.client.report [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1581.191030] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.343s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1581.191581] env[63273]: DEBUG nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1581.227227] env[63273]: DEBUG nova.compute.utils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1581.229088] env[63273]: DEBUG nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1581.229300] env[63273]: DEBUG nova.network.neutron [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1581.241951] env[63273]: DEBUG nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1581.314207] env[63273]: DEBUG nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1581.330319] env[63273]: DEBUG nova.policy [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b9cfeeeafe8448ea85b67a6cd1c54e46', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5683e7f13d7540c1bda6b8e43f1428d0', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1581.340633] env[63273]: DEBUG nova.virt.hardware [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1581.340879] env[63273]: DEBUG nova.virt.hardware [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1581.341052] env[63273]: DEBUG nova.virt.hardware [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1581.341278] env[63273]: DEBUG nova.virt.hardware [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1581.341432] env[63273]: DEBUG nova.virt.hardware [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1581.341581] env[63273]: DEBUG nova.virt.hardware [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1581.341793] env[63273]: DEBUG nova.virt.hardware [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1581.341955] env[63273]: DEBUG nova.virt.hardware [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1581.342136] env[63273]: DEBUG nova.virt.hardware [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1581.342303] env[63273]: DEBUG nova.virt.hardware [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1581.342478] env[63273]: DEBUG nova.virt.hardware [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1581.343410] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f81ee7e-464e-433b-85ca-77f66031588c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1581.352493] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f47e592-02e7-48cc-aeda-affbc53aafbe {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1581.829869] env[63273]: DEBUG nova.network.neutron [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Successfully created port: 88243553-1ee2-405f-afc4-762c1f92fa59 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1582.632471] env[63273]: DEBUG nova.compute.manager [req-924c0158-b639-419d-b676-e9f0e6b3c9c0 req-bc890ecc-681b-457c-becf-8b743751bd64 service nova] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Received event network-vif-plugged-88243553-1ee2-405f-afc4-762c1f92fa59 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1582.633118] env[63273]: DEBUG oslo_concurrency.lockutils [req-924c0158-b639-419d-b676-e9f0e6b3c9c0 req-bc890ecc-681b-457c-becf-8b743751bd64 service nova] Acquiring lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1582.633215] env[63273]: DEBUG oslo_concurrency.lockutils [req-924c0158-b639-419d-b676-e9f0e6b3c9c0 req-bc890ecc-681b-457c-becf-8b743751bd64 service nova] Lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1582.633647] env[63273]: DEBUG oslo_concurrency.lockutils [req-924c0158-b639-419d-b676-e9f0e6b3c9c0 req-bc890ecc-681b-457c-becf-8b743751bd64 service nova] Lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1582.634196] env[63273]: DEBUG nova.compute.manager [req-924c0158-b639-419d-b676-e9f0e6b3c9c0 req-bc890ecc-681b-457c-becf-8b743751bd64 service nova] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] No waiting events found dispatching network-vif-plugged-88243553-1ee2-405f-afc4-762c1f92fa59 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1582.634398] env[63273]: WARNING nova.compute.manager [req-924c0158-b639-419d-b676-e9f0e6b3c9c0 req-bc890ecc-681b-457c-becf-8b743751bd64 service nova] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Received unexpected event network-vif-plugged-88243553-1ee2-405f-afc4-762c1f92fa59 for instance with vm_state building and task_state spawning. [ 1582.705393] env[63273]: DEBUG nova.network.neutron [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Successfully updated port: 88243553-1ee2-405f-afc4-762c1f92fa59 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1582.719522] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "refresh_cache-c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1582.719717] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquired lock "refresh_cache-c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1582.719928] env[63273]: DEBUG nova.network.neutron [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1582.770187] env[63273]: DEBUG nova.network.neutron [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1582.995900] env[63273]: DEBUG nova.network.neutron [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Updating instance_info_cache with network_info: [{"id": "88243553-1ee2-405f-afc4-762c1f92fa59", "address": "fa:16:3e:b1:b8:5e", "network": {"id": "a7f8a09b-70fc-4a63-89a4-585b22da5db6", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-896645902-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5683e7f13d7540c1bda6b8e43f1428d0", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "bc9714ff-7109-4ea1-9435-b2b3fbdb9e81", "external-id": "nsx-vlan-transportzone-887", "segmentation_id": 887, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap88243553-1e", "ovs_interfaceid": "88243553-1ee2-405f-afc4-762c1f92fa59", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1583.007432] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Releasing lock "refresh_cache-c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1583.007736] env[63273]: DEBUG nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Instance network_info: |[{"id": "88243553-1ee2-405f-afc4-762c1f92fa59", "address": "fa:16:3e:b1:b8:5e", "network": {"id": "a7f8a09b-70fc-4a63-89a4-585b22da5db6", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-896645902-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5683e7f13d7540c1bda6b8e43f1428d0", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "bc9714ff-7109-4ea1-9435-b2b3fbdb9e81", "external-id": "nsx-vlan-transportzone-887", "segmentation_id": 887, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap88243553-1e", "ovs_interfaceid": "88243553-1ee2-405f-afc4-762c1f92fa59", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1583.008170] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:b1:b8:5e', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'bc9714ff-7109-4ea1-9435-b2b3fbdb9e81', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '88243553-1ee2-405f-afc4-762c1f92fa59', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1583.016575] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Creating folder: Project (5683e7f13d7540c1bda6b8e43f1428d0). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1583.017396] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d48f3782-1ba6-4147-9a78-ce36eaae59d6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1583.030306] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Created folder: Project (5683e7f13d7540c1bda6b8e43f1428d0) in parent group-v986930. [ 1583.030555] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Creating folder: Instances. Parent ref: group-v987023. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1583.030824] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-fe5d8aa0-0116-4579-bdc7-823f6e61a796 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1583.040762] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Created folder: Instances in parent group-v987023. [ 1583.040999] env[63273]: DEBUG oslo.service.loopingcall [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1583.041235] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1583.041449] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-a0accf22-c064-4b56-bf69-ce760f0f4858 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1583.062607] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1583.062607] env[63273]: value = "task-5072147" [ 1583.062607] env[63273]: _type = "Task" [ 1583.062607] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1583.070593] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072147, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1583.572258] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072147, 'name': CreateVM_Task, 'duration_secs': 0.310016} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1583.572462] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1583.573150] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1583.573319] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1583.573642] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1583.573906] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8534eaa4-71a9-4ebd-bf4c-830267bd5f0a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1583.578643] env[63273]: DEBUG oslo_vmware.api [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for the task: (returnval){ [ 1583.578643] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52d13a50-4e49-acd5-5bca-fe8be8fff0f7" [ 1583.578643] env[63273]: _type = "Task" [ 1583.578643] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1583.587763] env[63273]: DEBUG oslo_vmware.api [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52d13a50-4e49-acd5-5bca-fe8be8fff0f7, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1584.088981] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1584.089307] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1584.089548] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1584.657944] env[63273]: DEBUG nova.compute.manager [req-6bd5a7e4-6230-4bc5-95c3-939b16e9bd8f req-0ddfe433-0ad7-4e85-8269-162cddcd8777 service nova] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Received event network-changed-88243553-1ee2-405f-afc4-762c1f92fa59 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1584.658259] env[63273]: DEBUG nova.compute.manager [req-6bd5a7e4-6230-4bc5-95c3-939b16e9bd8f req-0ddfe433-0ad7-4e85-8269-162cddcd8777 service nova] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Refreshing instance network info cache due to event network-changed-88243553-1ee2-405f-afc4-762c1f92fa59. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1584.658566] env[63273]: DEBUG oslo_concurrency.lockutils [req-6bd5a7e4-6230-4bc5-95c3-939b16e9bd8f req-0ddfe433-0ad7-4e85-8269-162cddcd8777 service nova] Acquiring lock "refresh_cache-c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1584.658775] env[63273]: DEBUG oslo_concurrency.lockutils [req-6bd5a7e4-6230-4bc5-95c3-939b16e9bd8f req-0ddfe433-0ad7-4e85-8269-162cddcd8777 service nova] Acquired lock "refresh_cache-c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1584.658997] env[63273]: DEBUG nova.network.neutron [req-6bd5a7e4-6230-4bc5-95c3-939b16e9bd8f req-0ddfe433-0ad7-4e85-8269-162cddcd8777 service nova] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Refreshing network info cache for port 88243553-1ee2-405f-afc4-762c1f92fa59 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1584.971578] env[63273]: DEBUG nova.network.neutron [req-6bd5a7e4-6230-4bc5-95c3-939b16e9bd8f req-0ddfe433-0ad7-4e85-8269-162cddcd8777 service nova] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Updated VIF entry in instance network info cache for port 88243553-1ee2-405f-afc4-762c1f92fa59. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1584.971963] env[63273]: DEBUG nova.network.neutron [req-6bd5a7e4-6230-4bc5-95c3-939b16e9bd8f req-0ddfe433-0ad7-4e85-8269-162cddcd8777 service nova] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Updating instance_info_cache with network_info: [{"id": "88243553-1ee2-405f-afc4-762c1f92fa59", "address": "fa:16:3e:b1:b8:5e", "network": {"id": "a7f8a09b-70fc-4a63-89a4-585b22da5db6", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-896645902-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5683e7f13d7540c1bda6b8e43f1428d0", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "bc9714ff-7109-4ea1-9435-b2b3fbdb9e81", "external-id": "nsx-vlan-transportzone-887", "segmentation_id": 887, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap88243553-1e", "ovs_interfaceid": "88243553-1ee2-405f-afc4-762c1f92fa59", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1584.982312] env[63273]: DEBUG oslo_concurrency.lockutils [req-6bd5a7e4-6230-4bc5-95c3-939b16e9bd8f req-0ddfe433-0ad7-4e85-8269-162cddcd8777 service nova] Releasing lock "refresh_cache-c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1590.891389] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1591.891884] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1591.904251] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1591.904491] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1591.904646] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1591.904799] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1591.906216] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-baeabf41-31eb-475e-a320-28816909b228 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1591.914884] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43e023c2-7c80-4c81-88f6-524c7e717a7b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1591.928868] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-877d50d3-0e63-40e2-bd67-c41893020a03 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1591.935710] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59d11086-c310-40be-8d55-3600b5692770 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1591.965850] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180544MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1591.966055] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1591.966260] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1592.039243] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1592.039465] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1592.039566] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1592.039692] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1592.039814] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fa31b240-8bad-48ba-8339-155dc6acb265 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1592.039931] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1da174da-b4e6-437c-a538-53fc78b4282f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1592.040064] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance efe849d5-9df6-4813-a23b-c805e7eb2456 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1592.040231] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1592.040283] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 61bd6d66-590f-4512-afbf-9abf4a308749 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1592.040399] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1592.052772] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance dd885558-e9a6-490e-96fa-a4d82db45c5c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1592.063373] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92426c26-52fa-4978-a382-3684ec018bc5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1592.063609] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1592.063768] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '73', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '8', 'num_os_type_None': '10', 'num_proj_71b2d87fd7ff416db331d00f7375e59b': '2', 'io_workload': '10', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1', 'num_proj_6a6c31a7f8364be284dfcc2d167ed810': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_72f5eacfc9004694a5107a00edabbdd6': '1', 'num_proj_d277c5d52dd44fbdab7560c6fb34cf31': '1', 'num_task_spawning': '2', 'num_proj_a5420f4321c84264957a02c02f47649f': '1', 'num_proj_5683e7f13d7540c1bda6b8e43f1428d0': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1592.221050] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21f7db0a-c383-4965-bf76-3af720addcb0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1592.228924] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d8de605-f50f-415c-ba0e-39e321240259 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1592.259150] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e1c2503-8f92-4c00-bdbb-16b6bb4ab6ee {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1592.267019] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2773467d-c9c1-4701-bcc8-3ee2f30cf052 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1592.280070] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1592.289054] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1592.305653] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1592.305836] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.340s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1593.305858] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1594.886657] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1594.911721] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1596.892017] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1596.892435] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1596.892435] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1596.913827] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1596.913992] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1596.914247] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1596.914247] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1596.914367] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1596.914480] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1596.914600] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1596.914716] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1596.914830] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1596.914945] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1596.915070] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1597.891852] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1597.892103] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1598.892329] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1598.892661] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1599.546946] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "0afc321d-108c-4c2e-81a7-664091127047" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1599.547293] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "0afc321d-108c-4c2e-81a7-664091127047" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1600.905327] env[63273]: DEBUG oslo_concurrency.lockutils [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquiring lock "61bd6d66-590f-4512-afbf-9abf4a308749" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1602.886723] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1612.826389] env[63273]: DEBUG oslo_concurrency.lockutils [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1625.430201] env[63273]: WARNING oslo_vmware.rw_handles [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1625.430201] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1625.430201] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1625.430201] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1625.430201] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1625.430201] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1625.430201] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1625.430201] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1625.430201] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1625.430201] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1625.430201] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1625.430201] env[63273]: ERROR oslo_vmware.rw_handles [ 1625.430875] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/23dcf226-7a8c-4e8b-98c1-bf02296bcede/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1625.433286] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1625.433547] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Copying Virtual Disk [datastore1] vmware_temp/23dcf226-7a8c-4e8b-98c1-bf02296bcede/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/23dcf226-7a8c-4e8b-98c1-bf02296bcede/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1625.433832] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-06369ec4-b271-46e3-9691-af29d582018c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1625.441800] env[63273]: DEBUG oslo_vmware.api [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for the task: (returnval){ [ 1625.441800] env[63273]: value = "task-5072148" [ 1625.441800] env[63273]: _type = "Task" [ 1625.441800] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1625.449864] env[63273]: DEBUG oslo_vmware.api [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Task: {'id': task-5072148, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1625.953554] env[63273]: DEBUG oslo_vmware.exceptions [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1625.953803] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1625.954456] env[63273]: ERROR nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1625.954456] env[63273]: Faults: ['InvalidArgument'] [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Traceback (most recent call last): [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] yield resources [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] self.driver.spawn(context, instance, image_meta, [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] self._fetch_image_if_missing(context, vi) [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] image_cache(vi, tmp_image_ds_loc) [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] vm_util.copy_virtual_disk( [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] session._wait_for_task(vmdk_copy_task) [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] return self.wait_for_task(task_ref) [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] return evt.wait() [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] result = hub.switch() [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] return self.greenlet.switch() [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] self.f(*self.args, **self.kw) [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] raise exceptions.translate_fault(task_info.error) [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Faults: ['InvalidArgument'] [ 1625.954456] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] [ 1625.955691] env[63273]: INFO nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Terminating instance [ 1625.956417] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1625.956629] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1625.956899] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-6830c044-ea75-43ba-bd00-3d2045aa3eab {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1625.959121] env[63273]: DEBUG nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1625.959319] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1625.960055] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2dd3c452-c5f8-4ad2-9007-96cf0e990d0c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1625.967197] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1625.967462] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-da192072-2b8c-4006-83dc-d72077a2af63 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1625.969581] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1625.969738] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1625.970686] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-ddb5d7dc-db74-4742-a97e-ebad6ea4b434 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1625.975375] env[63273]: DEBUG oslo_vmware.api [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for the task: (returnval){ [ 1625.975375] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5280ccc2-7bf9-ebfb-3419-a0d06dc8d83e" [ 1625.975375] env[63273]: _type = "Task" [ 1625.975375] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1625.983521] env[63273]: DEBUG oslo_vmware.api [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5280ccc2-7bf9-ebfb-3419-a0d06dc8d83e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1626.034785] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1626.035069] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1626.035320] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Deleting the datastore file [datastore1] f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1626.035657] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-bb10ed9d-d4a9-4c5f-bfbd-44e2cad8957e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1626.043014] env[63273]: DEBUG oslo_vmware.api [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for the task: (returnval){ [ 1626.043014] env[63273]: value = "task-5072150" [ 1626.043014] env[63273]: _type = "Task" [ 1626.043014] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1626.051766] env[63273]: DEBUG oslo_vmware.api [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Task: {'id': task-5072150, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1626.485703] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1626.486060] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Creating directory with path [datastore1] vmware_temp/54697e3e-8b3b-484a-bcfb-09fa6acadf8e/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1626.486157] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-fd8464d5-ec35-412d-987c-c9ab8b42aced {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1626.498267] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Created directory with path [datastore1] vmware_temp/54697e3e-8b3b-484a-bcfb-09fa6acadf8e/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1626.498466] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Fetch image to [datastore1] vmware_temp/54697e3e-8b3b-484a-bcfb-09fa6acadf8e/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1626.498634] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/54697e3e-8b3b-484a-bcfb-09fa6acadf8e/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1626.499403] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85b97c01-ef6c-49be-bfd0-1fc24f274d24 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1626.506376] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afcc7192-5316-4f44-8a63-8bd9802b2e30 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1626.516031] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb178046-083c-4576-b95f-ac308724fbc8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1626.551026] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-277ce1f3-bb6e-4be4-a4e4-c59b1d7c2cea {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1626.557978] env[63273]: DEBUG oslo_vmware.api [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Task: {'id': task-5072150, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.08019} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1626.559526] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1626.559746] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1626.559931] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1626.560115] env[63273]: INFO nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1626.562208] env[63273]: DEBUG nova.compute.claims [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1626.562380] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1626.562595] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1626.565236] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-8be36770-b2f8-4aab-8d69-204105421bd4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1626.591477] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1626.650875] env[63273]: DEBUG oslo_vmware.rw_handles [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/54697e3e-8b3b-484a-bcfb-09fa6acadf8e/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1626.710558] env[63273]: DEBUG oslo_vmware.rw_handles [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1626.710754] env[63273]: DEBUG oslo_vmware.rw_handles [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/54697e3e-8b3b-484a-bcfb-09fa6acadf8e/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1626.847753] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47c5e80d-099d-4829-8e7a-e9f5202152d5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1626.856313] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96a981d5-1197-4798-b74d-54a0d3db362f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1626.887239] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e200e85-dc12-45e3-b755-e7e075879857 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1626.895291] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9328db01-b77a-406a-9dc1-3d8e3fcbf731 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1626.909119] env[63273]: DEBUG nova.compute.provider_tree [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1626.920055] env[63273]: DEBUG nova.scheduler.client.report [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1626.936848] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.374s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1626.937418] env[63273]: ERROR nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1626.937418] env[63273]: Faults: ['InvalidArgument'] [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Traceback (most recent call last): [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] self.driver.spawn(context, instance, image_meta, [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] self._fetch_image_if_missing(context, vi) [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] image_cache(vi, tmp_image_ds_loc) [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] vm_util.copy_virtual_disk( [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] session._wait_for_task(vmdk_copy_task) [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] return self.wait_for_task(task_ref) [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] return evt.wait() [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] result = hub.switch() [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] return self.greenlet.switch() [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] self.f(*self.args, **self.kw) [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] raise exceptions.translate_fault(task_info.error) [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Faults: ['InvalidArgument'] [ 1626.937418] env[63273]: ERROR nova.compute.manager [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] [ 1626.938423] env[63273]: DEBUG nova.compute.utils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1626.939698] env[63273]: DEBUG nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Build of instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb was re-scheduled: A specified parameter was not correct: fileType [ 1626.939698] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1626.940272] env[63273]: DEBUG nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1626.940451] env[63273]: DEBUG nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1626.940623] env[63273]: DEBUG nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1626.940786] env[63273]: DEBUG nova.network.neutron [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1627.198603] env[63273]: DEBUG nova.network.neutron [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1627.212024] env[63273]: INFO nova.compute.manager [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Took 0.27 seconds to deallocate network for instance. [ 1627.310973] env[63273]: INFO nova.scheduler.client.report [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Deleted allocations for instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb [ 1627.334102] env[63273]: DEBUG oslo_concurrency.lockutils [None req-5b416b7b-24e0-4f89-a8b0-0f9efbb11020 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 520.879s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1627.335342] env[63273]: DEBUG oslo_concurrency.lockutils [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 128.129s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1627.335576] env[63273]: DEBUG oslo_concurrency.lockutils [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1627.335783] env[63273]: DEBUG oslo_concurrency.lockutils [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1627.335986] env[63273]: DEBUG oslo_concurrency.lockutils [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1627.338773] env[63273]: INFO nova.compute.manager [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Terminating instance [ 1627.340465] env[63273]: DEBUG nova.compute.manager [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1627.341104] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1627.341216] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d8a54fc7-ee71-4911-bfd2-9adf30213843 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1627.351994] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aee7bf9c-542d-4fe0-ab87-439cae34ccf3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1627.362263] env[63273]: DEBUG nova.compute.manager [None req-918a1ecb-50e3-42f4-963c-d4a8ba6b8eac tempest-ServerPasswordTestJSON-1267960498 tempest-ServerPasswordTestJSON-1267960498-project-member] [instance: dd885558-e9a6-490e-96fa-a4d82db45c5c] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1627.385589] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb could not be found. [ 1627.385805] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1627.385980] env[63273]: INFO nova.compute.manager [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1627.386237] env[63273]: DEBUG oslo.service.loopingcall [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1627.386986] env[63273]: DEBUG nova.compute.manager [-] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1627.387102] env[63273]: DEBUG nova.network.neutron [-] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1627.388905] env[63273]: DEBUG nova.compute.manager [None req-918a1ecb-50e3-42f4-963c-d4a8ba6b8eac tempest-ServerPasswordTestJSON-1267960498 tempest-ServerPasswordTestJSON-1267960498-project-member] [instance: dd885558-e9a6-490e-96fa-a4d82db45c5c] Instance disappeared before build. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2431}} [ 1627.409883] env[63273]: DEBUG oslo_concurrency.lockutils [None req-918a1ecb-50e3-42f4-963c-d4a8ba6b8eac tempest-ServerPasswordTestJSON-1267960498 tempest-ServerPasswordTestJSON-1267960498-project-member] Lock "dd885558-e9a6-490e-96fa-a4d82db45c5c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 202.247s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1627.412974] env[63273]: DEBUG nova.network.neutron [-] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1627.419267] env[63273]: DEBUG nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1627.422717] env[63273]: INFO nova.compute.manager [-] [instance: f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb] Took 0.04 seconds to deallocate network for instance. [ 1627.482022] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1627.482022] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1627.482022] env[63273]: INFO nova.compute.claims [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1627.522837] env[63273]: DEBUG oslo_concurrency.lockutils [None req-465d5073-cb54-43f9-a4cf-4dcbf5f9ca2e tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "f3346e1d-96bd-4dd6-be14-2b5b87d3d7bb" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.187s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1627.721156] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70830de5-9f98-4b42-94b6-4284bc297b24 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1627.732050] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-605982be-205c-4e0b-80ca-646d1d2c0c20 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1627.760108] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0e6db91-9e92-470b-87f4-03155a47942a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1627.768540] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ec997ca-7cb8-4f6d-81f1-eadda52e831d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1627.783370] env[63273]: DEBUG nova.compute.provider_tree [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1627.792894] env[63273]: DEBUG nova.scheduler.client.report [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1627.811836] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.332s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1627.812369] env[63273]: DEBUG nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1627.850364] env[63273]: DEBUG nova.compute.utils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1627.851787] env[63273]: DEBUG nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1627.851972] env[63273]: DEBUG nova.network.neutron [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1627.862619] env[63273]: DEBUG nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1627.937492] env[63273]: DEBUG nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1627.964803] env[63273]: DEBUG nova.virt.hardware [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1627.965070] env[63273]: DEBUG nova.virt.hardware [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1627.965238] env[63273]: DEBUG nova.virt.hardware [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1627.965426] env[63273]: DEBUG nova.virt.hardware [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1627.965578] env[63273]: DEBUG nova.virt.hardware [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1627.965728] env[63273]: DEBUG nova.virt.hardware [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1627.965940] env[63273]: DEBUG nova.virt.hardware [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1627.966124] env[63273]: DEBUG nova.virt.hardware [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1627.966293] env[63273]: DEBUG nova.virt.hardware [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1627.966455] env[63273]: DEBUG nova.virt.hardware [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1627.966628] env[63273]: DEBUG nova.virt.hardware [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1627.967486] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7dd698e3-df07-4eff-9503-cd6fddbae293 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1627.976285] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de98136e-82ba-46db-83a2-31cf19459f5d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1628.157554] env[63273]: DEBUG nova.policy [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3df15315063d47baa6c32e2a3283e079', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '846931e260f34a22a2894c7cb88722fa', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1628.762886] env[63273]: DEBUG nova.network.neutron [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Successfully created port: 4a605213-88e7-4e4b-bca6-297d3b174ab4 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1630.082778] env[63273]: DEBUG nova.compute.manager [req-43a0ad52-e1a2-4ac0-b03c-4970698dcff0 req-9bd26be1-b09b-46e2-92ec-40e6f7690816 service nova] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Received event network-vif-plugged-4a605213-88e7-4e4b-bca6-297d3b174ab4 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1630.083087] env[63273]: DEBUG oslo_concurrency.lockutils [req-43a0ad52-e1a2-4ac0-b03c-4970698dcff0 req-9bd26be1-b09b-46e2-92ec-40e6f7690816 service nova] Acquiring lock "92426c26-52fa-4978-a382-3684ec018bc5-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1630.083261] env[63273]: DEBUG oslo_concurrency.lockutils [req-43a0ad52-e1a2-4ac0-b03c-4970698dcff0 req-9bd26be1-b09b-46e2-92ec-40e6f7690816 service nova] Lock "92426c26-52fa-4978-a382-3684ec018bc5-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1630.083414] env[63273]: DEBUG oslo_concurrency.lockutils [req-43a0ad52-e1a2-4ac0-b03c-4970698dcff0 req-9bd26be1-b09b-46e2-92ec-40e6f7690816 service nova] Lock "92426c26-52fa-4978-a382-3684ec018bc5-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1630.083580] env[63273]: DEBUG nova.compute.manager [req-43a0ad52-e1a2-4ac0-b03c-4970698dcff0 req-9bd26be1-b09b-46e2-92ec-40e6f7690816 service nova] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] No waiting events found dispatching network-vif-plugged-4a605213-88e7-4e4b-bca6-297d3b174ab4 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1630.083747] env[63273]: WARNING nova.compute.manager [req-43a0ad52-e1a2-4ac0-b03c-4970698dcff0 req-9bd26be1-b09b-46e2-92ec-40e6f7690816 service nova] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Received unexpected event network-vif-plugged-4a605213-88e7-4e4b-bca6-297d3b174ab4 for instance with vm_state building and task_state spawning. [ 1630.432299] env[63273]: DEBUG nova.network.neutron [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Successfully updated port: 4a605213-88e7-4e4b-bca6-297d3b174ab4 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1630.443526] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquiring lock "refresh_cache-92426c26-52fa-4978-a382-3684ec018bc5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1630.443677] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquired lock "refresh_cache-92426c26-52fa-4978-a382-3684ec018bc5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1630.443834] env[63273]: DEBUG nova.network.neutron [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1630.541910] env[63273]: DEBUG nova.network.neutron [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1630.755011] env[63273]: DEBUG nova.network.neutron [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Updating instance_info_cache with network_info: [{"id": "4a605213-88e7-4e4b-bca6-297d3b174ab4", "address": "fa:16:3e:b0:5f:46", "network": {"id": "4d1e98a2-4606-428f-9737-ebf5a21b56c1", "bridge": "br-int", "label": "tempest-ServerRescueTestJSONUnderV235-761732538-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": false}}], "meta": {"injected": false, "tenant_id": "846931e260f34a22a2894c7cb88722fa", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b4d548e7-d762-406a-bb2d-dc7168a8ca67", "external-id": "nsx-vlan-transportzone-796", "segmentation_id": 796, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4a605213-88", "ovs_interfaceid": "4a605213-88e7-4e4b-bca6-297d3b174ab4", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1630.768922] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Releasing lock "refresh_cache-92426c26-52fa-4978-a382-3684ec018bc5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1630.769230] env[63273]: DEBUG nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Instance network_info: |[{"id": "4a605213-88e7-4e4b-bca6-297d3b174ab4", "address": "fa:16:3e:b0:5f:46", "network": {"id": "4d1e98a2-4606-428f-9737-ebf5a21b56c1", "bridge": "br-int", "label": "tempest-ServerRescueTestJSONUnderV235-761732538-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": false}}], "meta": {"injected": false, "tenant_id": "846931e260f34a22a2894c7cb88722fa", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b4d548e7-d762-406a-bb2d-dc7168a8ca67", "external-id": "nsx-vlan-transportzone-796", "segmentation_id": 796, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4a605213-88", "ovs_interfaceid": "4a605213-88e7-4e4b-bca6-297d3b174ab4", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1630.769670] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:b0:5f:46', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'b4d548e7-d762-406a-bb2d-dc7168a8ca67', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '4a605213-88e7-4e4b-bca6-297d3b174ab4', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1630.777480] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Creating folder: Project (846931e260f34a22a2894c7cb88722fa). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1630.778098] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-005b774a-fe06-4d6e-a9e6-56234e08abf5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1630.790401] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Created folder: Project (846931e260f34a22a2894c7cb88722fa) in parent group-v986930. [ 1630.790605] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Creating folder: Instances. Parent ref: group-v987026. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1630.790856] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-1e2c56c1-bc82-45f8-9000-7c277dc56f09 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1630.800775] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Created folder: Instances in parent group-v987026. [ 1630.801077] env[63273]: DEBUG oslo.service.loopingcall [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1630.801223] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1630.801446] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-4c456814-b2f2-4a81-97e7-256d3a1c6ece {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1630.823162] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1630.823162] env[63273]: value = "task-5072153" [ 1630.823162] env[63273]: _type = "Task" [ 1630.823162] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1630.829795] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072153, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1631.334109] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072153, 'name': CreateVM_Task, 'duration_secs': 0.297284} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1631.334558] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1631.334849] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1631.335039] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1631.335376] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1631.335646] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-ef0bbaa7-7bdb-4ce0-9739-5185dc8736b8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1631.341353] env[63273]: DEBUG oslo_vmware.api [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Waiting for the task: (returnval){ [ 1631.341353] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]521827d2-835b-08f7-b429-b4931ed09517" [ 1631.341353] env[63273]: _type = "Task" [ 1631.341353] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1631.350444] env[63273]: DEBUG oslo_vmware.api [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]521827d2-835b-08f7-b429-b4931ed09517, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1631.853403] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1631.853737] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1631.854031] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1632.161419] env[63273]: DEBUG nova.compute.manager [req-cd2ca82f-8474-462b-b4cb-fb7dab12b590 req-85b2b9be-e916-481a-8611-ecf6c991536c service nova] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Received event network-changed-4a605213-88e7-4e4b-bca6-297d3b174ab4 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1632.161628] env[63273]: DEBUG nova.compute.manager [req-cd2ca82f-8474-462b-b4cb-fb7dab12b590 req-85b2b9be-e916-481a-8611-ecf6c991536c service nova] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Refreshing instance network info cache due to event network-changed-4a605213-88e7-4e4b-bca6-297d3b174ab4. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1632.164266] env[63273]: DEBUG oslo_concurrency.lockutils [req-cd2ca82f-8474-462b-b4cb-fb7dab12b590 req-85b2b9be-e916-481a-8611-ecf6c991536c service nova] Acquiring lock "refresh_cache-92426c26-52fa-4978-a382-3684ec018bc5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1632.164500] env[63273]: DEBUG oslo_concurrency.lockutils [req-cd2ca82f-8474-462b-b4cb-fb7dab12b590 req-85b2b9be-e916-481a-8611-ecf6c991536c service nova] Acquired lock "refresh_cache-92426c26-52fa-4978-a382-3684ec018bc5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1632.164688] env[63273]: DEBUG nova.network.neutron [req-cd2ca82f-8474-462b-b4cb-fb7dab12b590 req-85b2b9be-e916-481a-8611-ecf6c991536c service nova] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Refreshing network info cache for port 4a605213-88e7-4e4b-bca6-297d3b174ab4 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1632.600032] env[63273]: DEBUG nova.network.neutron [req-cd2ca82f-8474-462b-b4cb-fb7dab12b590 req-85b2b9be-e916-481a-8611-ecf6c991536c service nova] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Updated VIF entry in instance network info cache for port 4a605213-88e7-4e4b-bca6-297d3b174ab4. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1632.600509] env[63273]: DEBUG nova.network.neutron [req-cd2ca82f-8474-462b-b4cb-fb7dab12b590 req-85b2b9be-e916-481a-8611-ecf6c991536c service nova] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Updating instance_info_cache with network_info: [{"id": "4a605213-88e7-4e4b-bca6-297d3b174ab4", "address": "fa:16:3e:b0:5f:46", "network": {"id": "4d1e98a2-4606-428f-9737-ebf5a21b56c1", "bridge": "br-int", "label": "tempest-ServerRescueTestJSONUnderV235-761732538-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": false}}], "meta": {"injected": false, "tenant_id": "846931e260f34a22a2894c7cb88722fa", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b4d548e7-d762-406a-bb2d-dc7168a8ca67", "external-id": "nsx-vlan-transportzone-796", "segmentation_id": 796, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4a605213-88", "ovs_interfaceid": "4a605213-88e7-4e4b-bca6-297d3b174ab4", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1632.610711] env[63273]: DEBUG oslo_concurrency.lockutils [req-cd2ca82f-8474-462b-b4cb-fb7dab12b590 req-85b2b9be-e916-481a-8611-ecf6c991536c service nova] Releasing lock "refresh_cache-92426c26-52fa-4978-a382-3684ec018bc5" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1634.994803] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "2054144d-f068-4d52-b913-617b30d2d1c0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1634.996051] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "2054144d-f068-4d52-b913-617b30d2d1c0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1651.891631] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1651.892085] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1651.903016] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1651.903258] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1651.903441] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1651.903598] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1651.905127] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f3f8979-b142-456a-ab13-9f8859217296 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1651.914584] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a33131b-aa24-4122-95a5-b79f5ffdfa2c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1651.929447] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ac8d0a2-0b59-46e7-83e2-8863d64b6d18 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1651.936601] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a6d7196-278b-4033-a245-23806093589a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1651.966333] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180539MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1651.966524] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1651.966684] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1652.040124] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1652.040300] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1652.040514] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1652.040698] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fa31b240-8bad-48ba-8339-155dc6acb265 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1652.040835] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1da174da-b4e6-437c-a538-53fc78b4282f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1652.040959] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance efe849d5-9df6-4813-a23b-c805e7eb2456 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1652.041088] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1652.041207] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 61bd6d66-590f-4512-afbf-9abf4a308749 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1652.041326] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1652.041438] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92426c26-52fa-4978-a382-3684ec018bc5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1652.052782] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0afc321d-108c-4c2e-81a7-664091127047 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1652.063829] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2054144d-f068-4d52-b913-617b30d2d1c0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1652.064077] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1652.064212] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '75', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '9', 'num_os_type_None': '10', 'num_proj_71b2d87fd7ff416db331d00f7375e59b': '1', 'io_workload': '10', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1', 'num_proj_6a6c31a7f8364be284dfcc2d167ed810': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_72f5eacfc9004694a5107a00edabbdd6': '1', 'num_proj_d277c5d52dd44fbdab7560c6fb34cf31': '1', 'num_proj_a5420f4321c84264957a02c02f47649f': '1', 'num_proj_5683e7f13d7540c1bda6b8e43f1428d0': '1', 'num_task_spawning': '1', 'num_proj_846931e260f34a22a2894c7cb88722fa': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1652.223059] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2eba7b50-40d8-41c6-9010-4d06b42026dd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1652.230835] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44a6ade0-ae65-46c8-8946-814b0705cfc4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1652.259926] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ceaaa49-a72b-4d21-9203-d96b9778b8b1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1652.267549] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fcdec406-db93-46e5-b081-c066aaaf1fdc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1652.280631] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1652.289537] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1652.303865] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1652.304063] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.337s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1655.304060] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1655.891396] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1657.891288] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1658.892449] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1658.892727] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1658.892767] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1658.913484] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1658.913683] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1658.913828] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1658.913956] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1658.914091] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1658.914216] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1658.914333] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1658.914458] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1658.914578] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1658.914686] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1658.914807] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1658.915655] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1659.891712] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1659.891712] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1663.887822] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1673.606885] env[63273]: WARNING oslo_vmware.rw_handles [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1673.606885] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1673.606885] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1673.606885] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1673.606885] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1673.606885] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1673.606885] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1673.606885] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1673.606885] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1673.606885] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1673.606885] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1673.606885] env[63273]: ERROR oslo_vmware.rw_handles [ 1673.607672] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/54697e3e-8b3b-484a-bcfb-09fa6acadf8e/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1673.609538] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1673.609796] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Copying Virtual Disk [datastore1] vmware_temp/54697e3e-8b3b-484a-bcfb-09fa6acadf8e/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/54697e3e-8b3b-484a-bcfb-09fa6acadf8e/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1673.610134] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-72bebbeb-d14f-4a2a-b709-4d6bd5d17cee {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1673.619961] env[63273]: DEBUG oslo_vmware.api [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for the task: (returnval){ [ 1673.619961] env[63273]: value = "task-5072154" [ 1673.619961] env[63273]: _type = "Task" [ 1673.619961] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1673.627702] env[63273]: DEBUG oslo_vmware.api [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Task: {'id': task-5072154, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1674.131056] env[63273]: DEBUG oslo_vmware.exceptions [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1674.131305] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1674.131880] env[63273]: ERROR nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1674.131880] env[63273]: Faults: ['InvalidArgument'] [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Traceback (most recent call last): [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] yield resources [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] self.driver.spawn(context, instance, image_meta, [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] self._fetch_image_if_missing(context, vi) [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] image_cache(vi, tmp_image_ds_loc) [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] vm_util.copy_virtual_disk( [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] session._wait_for_task(vmdk_copy_task) [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] return self.wait_for_task(task_ref) [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] return evt.wait() [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] result = hub.switch() [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] return self.greenlet.switch() [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] self.f(*self.args, **self.kw) [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] raise exceptions.translate_fault(task_info.error) [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Faults: ['InvalidArgument'] [ 1674.131880] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] [ 1674.133021] env[63273]: INFO nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Terminating instance [ 1674.133934] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1674.134162] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1674.134420] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-777cb491-cccb-4467-a360-1e2961cc7b7f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.137047] env[63273]: DEBUG nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1674.137360] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1674.138176] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb3d1f5d-c9c3-4854-83b2-06e68bfd96aa {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.145932] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1674.146268] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-3c9463e7-35f7-401b-a2e5-5d077e130b01 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.148762] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1674.148996] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1674.150051] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-018b7201-4beb-4ac3-afd6-c9102b0f9922 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.155157] env[63273]: DEBUG oslo_vmware.api [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Waiting for the task: (returnval){ [ 1674.155157] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52b131cf-92c1-37d8-9f22-8de6d039b938" [ 1674.155157] env[63273]: _type = "Task" [ 1674.155157] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1674.170138] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1674.170465] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Creating directory with path [datastore1] vmware_temp/c7b4ec75-1d68-4147-b0b8-2acc8d0013df/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1674.170708] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a0d8ddca-56e3-41eb-b64c-662a2e0adcb0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.183929] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Created directory with path [datastore1] vmware_temp/c7b4ec75-1d68-4147-b0b8-2acc8d0013df/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1674.184198] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Fetch image to [datastore1] vmware_temp/c7b4ec75-1d68-4147-b0b8-2acc8d0013df/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1674.184340] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/c7b4ec75-1d68-4147-b0b8-2acc8d0013df/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1674.185194] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ee876f1-fb64-42e1-997b-1e66cf493ae1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.193351] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1a53f3e-c410-44d4-a44f-3be924166b0e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.204084] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45f170f3-2c58-4928-8897-6e67ad2cdc41 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.237893] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0dc0192f-5f2f-4517-9ca8-324f782d4655 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.241078] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1674.241078] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1674.241078] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Deleting the datastore file [datastore1] 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1674.241373] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-a13f3bde-3c73-438c-a85b-00638bc0f88e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.246948] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-df1b3357-e290-4595-9852-7632fe9aa404 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.250318] env[63273]: DEBUG oslo_vmware.api [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for the task: (returnval){ [ 1674.250318] env[63273]: value = "task-5072156" [ 1674.250318] env[63273]: _type = "Task" [ 1674.250318] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1674.259287] env[63273]: DEBUG oslo_vmware.api [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Task: {'id': task-5072156, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1674.274010] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1674.430574] env[63273]: DEBUG oslo_vmware.rw_handles [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/c7b4ec75-1d68-4147-b0b8-2acc8d0013df/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1674.488963] env[63273]: DEBUG oslo_vmware.rw_handles [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1674.489188] env[63273]: DEBUG oslo_vmware.rw_handles [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/c7b4ec75-1d68-4147-b0b8-2acc8d0013df/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1674.759918] env[63273]: DEBUG oslo_vmware.api [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Task: {'id': task-5072156, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.073657} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1674.760367] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1674.760431] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1674.760548] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1674.760726] env[63273]: INFO nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1674.762875] env[63273]: DEBUG nova.compute.claims [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1674.763083] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1674.763279] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1674.977200] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2815f1b7-65b9-4432-8761-d9db3c9e20d7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1674.984960] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85f86914-0304-452b-af4b-33298a34527b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1675.015372] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60f2e8dc-8733-4ebc-bdb7-1703958a30cf {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1675.023460] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b90e84b-3c57-4433-8f0e-0ed7f9f49e86 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1675.036942] env[63273]: DEBUG nova.compute.provider_tree [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1675.046558] env[63273]: DEBUG nova.scheduler.client.report [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1675.061292] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.297s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1675.061292] env[63273]: ERROR nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1675.061292] env[63273]: Faults: ['InvalidArgument'] [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Traceback (most recent call last): [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] self.driver.spawn(context, instance, image_meta, [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] self._fetch_image_if_missing(context, vi) [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] image_cache(vi, tmp_image_ds_loc) [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] vm_util.copy_virtual_disk( [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] session._wait_for_task(vmdk_copy_task) [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] return self.wait_for_task(task_ref) [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] return evt.wait() [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] result = hub.switch() [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] return self.greenlet.switch() [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] self.f(*self.args, **self.kw) [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] raise exceptions.translate_fault(task_info.error) [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Faults: ['InvalidArgument'] [ 1675.061292] env[63273]: ERROR nova.compute.manager [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] [ 1675.062826] env[63273]: DEBUG nova.compute.utils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1675.063624] env[63273]: DEBUG nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Build of instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 was re-scheduled: A specified parameter was not correct: fileType [ 1675.063624] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1675.064054] env[63273]: DEBUG nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1675.064225] env[63273]: DEBUG nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1675.064423] env[63273]: DEBUG nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1675.064620] env[63273]: DEBUG nova.network.neutron [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1675.400007] env[63273]: DEBUG nova.network.neutron [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1675.412517] env[63273]: INFO nova.compute.manager [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Took 0.35 seconds to deallocate network for instance. [ 1675.501973] env[63273]: INFO nova.scheduler.client.report [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Deleted allocations for instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 [ 1675.537480] env[63273]: DEBUG oslo_concurrency.lockutils [None req-b4584783-2838-42e2-a824-fc89fec00e0c tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 568.580s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1675.539304] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 372.845s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1675.539526] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Acquiring lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1675.539733] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1675.539897] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1675.541771] env[63273]: INFO nova.compute.manager [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Terminating instance [ 1675.544151] env[63273]: DEBUG nova.compute.manager [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1675.544480] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1675.544852] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1b198fde-6c47-455c-911c-41992208d5b2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1675.556921] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f81a1ec-d7a4-446d-b0a4-8224a839267c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1675.568383] env[63273]: DEBUG nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1675.593820] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5d5955e6-416d-4eaa-b23e-c886ba4d4f26 could not be found. [ 1675.594054] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1675.594214] env[63273]: INFO nova.compute.manager [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1675.594465] env[63273]: DEBUG oslo.service.loopingcall [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1675.594696] env[63273]: DEBUG nova.compute.manager [-] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1675.594793] env[63273]: DEBUG nova.network.neutron [-] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1675.620548] env[63273]: DEBUG nova.network.neutron [-] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1675.628839] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1675.629134] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1675.630768] env[63273]: INFO nova.compute.claims [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1675.635055] env[63273]: INFO nova.compute.manager [-] [instance: 5d5955e6-416d-4eaa-b23e-c886ba4d4f26] Took 0.04 seconds to deallocate network for instance. [ 1675.738513] env[63273]: DEBUG oslo_concurrency.lockutils [None req-6982463a-0cf1-4d46-821e-1b33387446e2 tempest-ServersAdminTestJSON-510963903 tempest-ServersAdminTestJSON-510963903-project-member] Lock "5d5955e6-416d-4eaa-b23e-c886ba4d4f26" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.199s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1675.840133] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-289e2e6f-0306-47dd-996d-67833f702340 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1675.847812] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86c9e517-ba2a-4aa6-a47e-e2991717b647 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1675.878207] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79465a48-7e1f-45fa-b43a-4ca5f9d72e16 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1675.886437] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e90ea44-37a0-4c2c-83b4-de533dc49cda {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1675.900251] env[63273]: DEBUG nova.compute.provider_tree [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1675.910437] env[63273]: DEBUG nova.scheduler.client.report [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1675.924759] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.296s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1675.925484] env[63273]: DEBUG nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1675.959772] env[63273]: DEBUG nova.compute.utils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1675.961588] env[63273]: DEBUG nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1675.961715] env[63273]: DEBUG nova.network.neutron [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1675.975175] env[63273]: DEBUG nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1676.040765] env[63273]: DEBUG nova.policy [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e1ab27df058544c496bd2a078b84e6e8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3387519998ea4ed8a586caeaafed2446', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1676.047025] env[63273]: DEBUG nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1676.072133] env[63273]: DEBUG nova.virt.hardware [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1676.072396] env[63273]: DEBUG nova.virt.hardware [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1676.072559] env[63273]: DEBUG nova.virt.hardware [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1676.072733] env[63273]: DEBUG nova.virt.hardware [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1676.072881] env[63273]: DEBUG nova.virt.hardware [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1676.073039] env[63273]: DEBUG nova.virt.hardware [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1676.073461] env[63273]: DEBUG nova.virt.hardware [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1676.073682] env[63273]: DEBUG nova.virt.hardware [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1676.073878] env[63273]: DEBUG nova.virt.hardware [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1676.074067] env[63273]: DEBUG nova.virt.hardware [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1676.074258] env[63273]: DEBUG nova.virt.hardware [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1676.075144] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f0dec06-e362-4eb2-bb29-d54bd34d1ac3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1676.083957] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56381d7d-fe3c-4e62-bd2f-b5e5ebfa341e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1676.398067] env[63273]: DEBUG nova.network.neutron [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Successfully created port: 6c663db4-b9b0-4f27-bdd8-3706d4033d3a {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1677.146092] env[63273]: DEBUG nova.network.neutron [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Successfully updated port: 6c663db4-b9b0-4f27-bdd8-3706d4033d3a {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1677.159030] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "refresh_cache-0afc321d-108c-4c2e-81a7-664091127047" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1677.159162] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired lock "refresh_cache-0afc321d-108c-4c2e-81a7-664091127047" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1677.159288] env[63273]: DEBUG nova.network.neutron [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1677.233786] env[63273]: DEBUG nova.network.neutron [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1677.479763] env[63273]: DEBUG nova.compute.manager [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Received event network-vif-plugged-6c663db4-b9b0-4f27-bdd8-3706d4033d3a {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1677.479986] env[63273]: DEBUG oslo_concurrency.lockutils [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] Acquiring lock "0afc321d-108c-4c2e-81a7-664091127047-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1677.480291] env[63273]: DEBUG oslo_concurrency.lockutils [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] Lock "0afc321d-108c-4c2e-81a7-664091127047-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1677.480443] env[63273]: DEBUG oslo_concurrency.lockutils [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] Lock "0afc321d-108c-4c2e-81a7-664091127047-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1677.480589] env[63273]: DEBUG nova.compute.manager [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] [instance: 0afc321d-108c-4c2e-81a7-664091127047] No waiting events found dispatching network-vif-plugged-6c663db4-b9b0-4f27-bdd8-3706d4033d3a {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1677.480799] env[63273]: WARNING nova.compute.manager [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Received unexpected event network-vif-plugged-6c663db4-b9b0-4f27-bdd8-3706d4033d3a for instance with vm_state building and task_state spawning. [ 1677.480982] env[63273]: DEBUG nova.compute.manager [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Received event network-changed-6c663db4-b9b0-4f27-bdd8-3706d4033d3a {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1677.481261] env[63273]: DEBUG nova.compute.manager [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Refreshing instance network info cache due to event network-changed-6c663db4-b9b0-4f27-bdd8-3706d4033d3a. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1677.481438] env[63273]: DEBUG oslo_concurrency.lockutils [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] Acquiring lock "refresh_cache-0afc321d-108c-4c2e-81a7-664091127047" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1677.691878] env[63273]: DEBUG nova.network.neutron [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Updating instance_info_cache with network_info: [{"id": "6c663db4-b9b0-4f27-bdd8-3706d4033d3a", "address": "fa:16:3e:1a:ca:c3", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6c663db4-b9", "ovs_interfaceid": "6c663db4-b9b0-4f27-bdd8-3706d4033d3a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1677.705935] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Releasing lock "refresh_cache-0afc321d-108c-4c2e-81a7-664091127047" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1677.706613] env[63273]: DEBUG nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Instance network_info: |[{"id": "6c663db4-b9b0-4f27-bdd8-3706d4033d3a", "address": "fa:16:3e:1a:ca:c3", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6c663db4-b9", "ovs_interfaceid": "6c663db4-b9b0-4f27-bdd8-3706d4033d3a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1677.707640] env[63273]: DEBUG oslo_concurrency.lockutils [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] Acquired lock "refresh_cache-0afc321d-108c-4c2e-81a7-664091127047" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1677.707640] env[63273]: DEBUG nova.network.neutron [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Refreshing network info cache for port 6c663db4-b9b0-4f27-bdd8-3706d4033d3a {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1677.711018] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:1a:ca:c3', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'dad4f433-bb0b-45c7-8040-972ef2277f75', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '6c663db4-b9b0-4f27-bdd8-3706d4033d3a', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1677.720930] env[63273]: DEBUG oslo.service.loopingcall [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1677.722016] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1677.724588] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-491741c9-6c37-4f57-bec5-d0842683e631 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1677.746864] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1677.746864] env[63273]: value = "task-5072157" [ 1677.746864] env[63273]: _type = "Task" [ 1677.746864] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1677.756153] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072157, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1678.053682] env[63273]: DEBUG nova.network.neutron [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Updated VIF entry in instance network info cache for port 6c663db4-b9b0-4f27-bdd8-3706d4033d3a. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1678.054072] env[63273]: DEBUG nova.network.neutron [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Updating instance_info_cache with network_info: [{"id": "6c663db4-b9b0-4f27-bdd8-3706d4033d3a", "address": "fa:16:3e:1a:ca:c3", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6c663db4-b9", "ovs_interfaceid": "6c663db4-b9b0-4f27-bdd8-3706d4033d3a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1678.064531] env[63273]: DEBUG oslo_concurrency.lockutils [req-b8a6383b-a0e9-44f4-baae-6d7ad1f08e7d req-7b86ecf2-dc01-400b-9949-8f77d9ce825b service nova] Releasing lock "refresh_cache-0afc321d-108c-4c2e-81a7-664091127047" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1678.257167] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072157, 'name': CreateVM_Task, 'duration_secs': 0.295616} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1678.257414] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1678.258170] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1678.258394] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1678.258712] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1678.258969] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2c8a9282-53c3-45ca-ad23-5ba0906ae1a9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1678.263850] env[63273]: DEBUG oslo_vmware.api [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 1678.263850] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52c2c0fd-c616-db40-700d-0741dcf3fb56" [ 1678.263850] env[63273]: _type = "Task" [ 1678.263850] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1678.271916] env[63273]: DEBUG oslo_vmware.api [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52c2c0fd-c616-db40-700d-0741dcf3fb56, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1678.775160] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1678.775385] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1678.775607] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1684.083889] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "7262f23a-8ee4-4468-8e49-2e9297352dc9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1684.084233] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "7262f23a-8ee4-4468-8e49-2e9297352dc9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1710.892082] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1712.900043] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1713.892348] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1713.904497] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1713.904812] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1713.904969] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1713.905196] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1713.906265] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-888a1d7e-4a38-4e13-87bf-5197feeb3018 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1713.916986] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81300146-c6e4-42cd-a2e7-db9039ca48ed {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1713.931804] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-081900c9-e393-4e8d-b5e5-ffd8ab8ee5e8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1713.938977] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91b333cc-97fd-46fb-94cd-815b891adb63 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1713.968506] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180541MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1713.968656] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1713.968864] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1714.048860] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1714.049032] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1714.049164] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fa31b240-8bad-48ba-8339-155dc6acb265 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1714.049287] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1da174da-b4e6-437c-a538-53fc78b4282f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1714.049407] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance efe849d5-9df6-4813-a23b-c805e7eb2456 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1714.049528] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1714.049645] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 61bd6d66-590f-4512-afbf-9abf4a308749 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1714.049759] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1714.049871] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92426c26-52fa-4978-a382-3684ec018bc5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1714.049982] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0afc321d-108c-4c2e-81a7-664091127047 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1714.066678] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2054144d-f068-4d52-b913-617b30d2d1c0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1714.077727] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1767}} [ 1714.077968] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1714.078163] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=100GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] stats={'failed_builds': '76', 'num_instances': '10', 'num_vm_building': '10', 'num_task_deleting': '8', 'num_os_type_None': '10', 'num_proj_702825b6bec4497f89ef4091bd7af0da': '1', 'io_workload': '10', 'num_proj_6a6c31a7f8364be284dfcc2d167ed810': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '2', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_proj_72f5eacfc9004694a5107a00edabbdd6': '1', 'num_proj_d277c5d52dd44fbdab7560c6fb34cf31': '1', 'num_proj_a5420f4321c84264957a02c02f47649f': '1', 'num_proj_5683e7f13d7540c1bda6b8e43f1428d0': '1', 'num_task_spawning': '2', 'num_proj_846931e260f34a22a2894c7cb88722fa': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1714.220900] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d732d990-4487-4185-8e4c-b0453e15be5a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.229355] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-008bd85d-2f8a-40bf-96ca-278f25912e76 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.260791] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1f12f0e-aa27-4c60-8a88-30674815e99a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.268815] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b80f3334-1e3a-472d-8a04-0326a2c5cbd5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.282323] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1714.290350] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1714.306043] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1714.306251] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.337s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1714.891593] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1714.892035] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1714.892206] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances with incomplete migration {{(pid=63273) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11345}} [ 1716.898063] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1717.891627] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1718.892037] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1718.892445] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1718.892445] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1718.912542] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1718.912713] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1718.912834] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1718.912960] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1718.913094] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1718.913216] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1718.913335] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1718.913466] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1718.913624] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1718.913760] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1718.913880] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1718.914381] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1718.914551] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1721.102677] env[63273]: WARNING oslo_vmware.rw_handles [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1721.102677] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1721.102677] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1721.102677] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1721.102677] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1721.102677] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1721.102677] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1721.102677] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1721.102677] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1721.102677] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1721.102677] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1721.102677] env[63273]: ERROR oslo_vmware.rw_handles [ 1721.103502] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/c7b4ec75-1d68-4147-b0b8-2acc8d0013df/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1721.105172] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1721.105456] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Copying Virtual Disk [datastore1] vmware_temp/c7b4ec75-1d68-4147-b0b8-2acc8d0013df/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/c7b4ec75-1d68-4147-b0b8-2acc8d0013df/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1721.105766] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-723ac579-b670-40b0-881d-206f086ff344 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1721.116219] env[63273]: DEBUG oslo_vmware.api [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Waiting for the task: (returnval){ [ 1721.116219] env[63273]: value = "task-5072158" [ 1721.116219] env[63273]: _type = "Task" [ 1721.116219] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1721.124651] env[63273]: DEBUG oslo_vmware.api [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Task: {'id': task-5072158, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1721.626791] env[63273]: DEBUG oslo_vmware.exceptions [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1721.627124] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1721.627701] env[63273]: ERROR nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1721.627701] env[63273]: Faults: ['InvalidArgument'] [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Traceback (most recent call last): [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] yield resources [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] self.driver.spawn(context, instance, image_meta, [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] self._fetch_image_if_missing(context, vi) [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] image_cache(vi, tmp_image_ds_loc) [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] vm_util.copy_virtual_disk( [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] session._wait_for_task(vmdk_copy_task) [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] return self.wait_for_task(task_ref) [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] return evt.wait() [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] result = hub.switch() [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] return self.greenlet.switch() [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] self.f(*self.args, **self.kw) [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] raise exceptions.translate_fault(task_info.error) [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Faults: ['InvalidArgument'] [ 1721.627701] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] [ 1721.628858] env[63273]: INFO nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Terminating instance [ 1721.629790] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1721.629879] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1721.630119] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-297166dc-c59d-4782-bb6d-9484f79bc594 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1721.632281] env[63273]: DEBUG nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1721.632518] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1721.633248] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-608be85a-2823-4422-8962-308bccf16135 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1721.640596] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1721.640911] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-f692c8b4-4c97-4e5a-8b62-7ae25eb2a3cd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1721.643252] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1721.643422] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1721.644449] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-88252cae-26b3-4e61-aa45-cd950238d468 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1721.650155] env[63273]: DEBUG oslo_vmware.api [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Waiting for the task: (returnval){ [ 1721.650155] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ef3dea-dad7-303b-9407-c452e154b72c" [ 1721.650155] env[63273]: _type = "Task" [ 1721.650155] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1721.659698] env[63273]: DEBUG oslo_vmware.api [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ef3dea-dad7-303b-9407-c452e154b72c, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1721.717045] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1721.717252] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1721.717380] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Deleting the datastore file [datastore1] c4c785e1-274c-4713-98e1-9d7e5e8249a0 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1721.717660] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-cf87c0ab-66ae-42a5-a774-99acab162bd4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1721.726397] env[63273]: DEBUG oslo_vmware.api [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Waiting for the task: (returnval){ [ 1721.726397] env[63273]: value = "task-5072160" [ 1721.726397] env[63273]: _type = "Task" [ 1721.726397] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1721.734951] env[63273]: DEBUG oslo_vmware.api [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Task: {'id': task-5072160, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1721.891785] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1721.891979] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1722.161211] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1722.161493] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Creating directory with path [datastore1] vmware_temp/e553e035-e94c-4d8b-969d-83b625b07d16/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1722.161739] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-70bc6380-ccf4-42c2-a247-4457f5d20f2f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.173620] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Created directory with path [datastore1] vmware_temp/e553e035-e94c-4d8b-969d-83b625b07d16/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1722.173814] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Fetch image to [datastore1] vmware_temp/e553e035-e94c-4d8b-969d-83b625b07d16/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1722.173985] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/e553e035-e94c-4d8b-969d-83b625b07d16/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1722.174751] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81e995f6-0abf-40ac-a255-466fdc322717 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.181466] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5ce0b39-be99-4fe7-aa61-c33300c333a9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.190848] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-050ab938-d8c6-46a5-a6b7-201423567e02 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.220962] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48732111-8213-4377-a7b4-1e36460df9d6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.230167] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-55ee897b-88db-4109-bcab-d81844e222cb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.237162] env[63273]: DEBUG oslo_vmware.api [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Task: {'id': task-5072160, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.070734} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1722.237386] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1722.237568] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1722.237767] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1722.237935] env[63273]: INFO nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Took 0.61 seconds to destroy the instance on the hypervisor. [ 1722.240202] env[63273]: DEBUG nova.compute.claims [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1722.240409] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1722.240627] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1722.253122] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1722.362412] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1722.363335] env[63273]: ERROR nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Instance failed to spawn: nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Traceback (most recent call last): [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] result = getattr(controller, method)(*args, **kwargs) [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._get(image_id) [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] resp, body = self.http_client.get(url, headers=header) [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.request(url, 'GET', **kwargs) [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._handle_response(resp) [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise exc.from_response(resp, resp.content) [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] During handling of the above exception, another exception occurred: [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Traceback (most recent call last): [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] yield resources [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self.driver.spawn(context, instance, image_meta, [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._fetch_image_if_missing(context, vi) [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] image_fetch(context, vi, tmp_image_ds_loc) [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] images.fetch_image( [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1722.363335] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] metadata = IMAGE_API.get(context, image_ref) [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return session.show(context, image_id, [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] _reraise_translated_image_exception(image_id) [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise new_exc.with_traceback(exc_trace) [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] result = getattr(controller, method)(*args, **kwargs) [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._get(image_id) [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] resp, body = self.http_client.get(url, headers=header) [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.request(url, 'GET', **kwargs) [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._handle_response(resp) [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise exc.from_response(resp, resp.content) [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1722.364603] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1722.364603] env[63273]: INFO nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Terminating instance [ 1722.365373] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1722.365373] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1722.366012] env[63273]: DEBUG nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1722.366204] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1722.366449] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-048813d4-e6c9-468c-9dd3-1d96b58d163c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.368974] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3ade40e-f7fa-4a8e-9b30-c7e2ba707888 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.377982] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1722.379092] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-6d930b2f-c208-487d-bc43-5b0c37886e44 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.380566] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1722.380754] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1722.381436] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-27ffe3e9-115e-4327-a5d4-c8fcee043268 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.387015] env[63273]: DEBUG oslo_vmware.api [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 1722.387015] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52671d6a-a817-20b3-0958-09e53de0566f" [ 1722.387015] env[63273]: _type = "Task" [ 1722.387015] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1722.399397] env[63273]: DEBUG oslo_vmware.api [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52671d6a-a817-20b3-0958-09e53de0566f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1722.477184] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1300056f-aecd-45d9-bc28-4c5985a9a498 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.482312] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1722.482312] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1722.482312] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Deleting the datastore file [datastore1] 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1722.482312] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-d3ee619e-3303-408c-a306-bf3f9b3c6602 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.488266] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fde6b6fc-7c1c-42b3-9c76-370afb64c0e8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.493093] env[63273]: DEBUG oslo_vmware.api [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Waiting for the task: (returnval){ [ 1722.493093] env[63273]: value = "task-5072162" [ 1722.493093] env[63273]: _type = "Task" [ 1722.493093] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1722.521111] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0414a2bf-a698-4fca-a985-37a2b70989dc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.526739] env[63273]: DEBUG oslo_vmware.api [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Task: {'id': task-5072162, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1722.532131] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb64bfa9-ea63-4d39-8e99-706d243b8476 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.545940] env[63273]: DEBUG nova.compute.provider_tree [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1722.554709] env[63273]: DEBUG nova.scheduler.client.report [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1722.569625] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.329s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1722.570122] env[63273]: ERROR nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1722.570122] env[63273]: Faults: ['InvalidArgument'] [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Traceback (most recent call last): [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] self.driver.spawn(context, instance, image_meta, [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] self._fetch_image_if_missing(context, vi) [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] image_cache(vi, tmp_image_ds_loc) [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] vm_util.copy_virtual_disk( [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] session._wait_for_task(vmdk_copy_task) [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] return self.wait_for_task(task_ref) [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] return evt.wait() [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] result = hub.switch() [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] return self.greenlet.switch() [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] self.f(*self.args, **self.kw) [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] raise exceptions.translate_fault(task_info.error) [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Faults: ['InvalidArgument'] [ 1722.570122] env[63273]: ERROR nova.compute.manager [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] [ 1722.571196] env[63273]: DEBUG nova.compute.utils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1722.572568] env[63273]: DEBUG nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Build of instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 was re-scheduled: A specified parameter was not correct: fileType [ 1722.572568] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1722.572957] env[63273]: DEBUG nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1722.573145] env[63273]: DEBUG nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1722.573315] env[63273]: DEBUG nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1722.573476] env[63273]: DEBUG nova.network.neutron [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1722.899758] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1722.900021] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Creating directory with path [datastore1] vmware_temp/2021caf8-c1ac-432c-a550-9f8966e20e6d/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1722.900321] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0cf6bc1b-c024-40c1-ac86-6e14716ef59e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.912153] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Created directory with path [datastore1] vmware_temp/2021caf8-c1ac-432c-a550-9f8966e20e6d/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1722.912356] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Fetch image to [datastore1] vmware_temp/2021caf8-c1ac-432c-a550-9f8966e20e6d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1722.912565] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/2021caf8-c1ac-432c-a550-9f8966e20e6d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1722.913378] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fadb9925-6f04-47ee-967f-e903d06f1828 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.921282] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8fac610-976e-4e6b-ba3d-a8b44bf2f5ff {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.932559] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1eca0a95-0681-4380-a1ed-3e3c29fed56a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.963632] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b198904-c5d3-4e26-9efe-c5481ff063e6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.970187] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-dd3e8106-b18a-4db1-82df-dd17f4390d60 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1722.999644] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1723.009718] env[63273]: DEBUG oslo_vmware.api [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Task: {'id': task-5072162, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.082888} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1723.009878] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1723.009878] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1723.010424] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1723.010424] env[63273]: INFO nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Took 0.64 seconds to destroy the instance on the hypervisor. [ 1723.015893] env[63273]: DEBUG nova.compute.claims [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1723.015893] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1723.015893] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1723.062495] env[63273]: DEBUG oslo_vmware.rw_handles [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/2021caf8-c1ac-432c-a550-9f8966e20e6d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1723.141987] env[63273]: DEBUG oslo_vmware.rw_handles [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1723.141987] env[63273]: DEBUG oslo_vmware.rw_handles [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/2021caf8-c1ac-432c-a550-9f8966e20e6d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1723.146930] env[63273]: DEBUG nova.network.neutron [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1723.160696] env[63273]: INFO nova.compute.manager [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Took 0.59 seconds to deallocate network for instance. [ 1723.284182] env[63273]: INFO nova.scheduler.client.report [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Deleted allocations for instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 [ 1723.313025] env[63273]: DEBUG oslo_concurrency.lockutils [None req-67a6be4c-24a3-4d8b-a935-71517d762e3b tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 548.343s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1723.314237] env[63273]: DEBUG oslo_concurrency.lockutils [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 352.276s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1723.314475] env[63273]: DEBUG oslo_concurrency.lockutils [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Acquiring lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1723.314687] env[63273]: DEBUG oslo_concurrency.lockutils [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1723.314883] env[63273]: DEBUG oslo_concurrency.lockutils [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1723.317096] env[63273]: INFO nova.compute.manager [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Terminating instance [ 1723.318974] env[63273]: DEBUG nova.compute.manager [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1723.319250] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1723.319797] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-116f43a6-e3df-4f61-aa9c-76a52f7ef523 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.324131] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68409d36-b66f-4bfb-b70d-6a27cf6b0a6e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.328628] env[63273]: DEBUG nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1723.337294] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd7ccf9c-8660-4153-bb72-0f804719e640 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.350526] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d8360ab-8d7e-4e2f-8b9d-4b86509dd204 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.396486] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa390cb6-79e6-48f0-8ba8-1ac50347e9e5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.399589] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c4c785e1-274c-4713-98e1-9d7e5e8249a0 could not be found. [ 1723.399803] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1723.399984] env[63273]: INFO nova.compute.manager [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Took 0.08 seconds to destroy the instance on the hypervisor. [ 1723.400243] env[63273]: DEBUG oslo.service.loopingcall [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1723.402464] env[63273]: DEBUG nova.compute.manager [-] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1723.402593] env[63273]: DEBUG nova.network.neutron [-] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1723.410125] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d064790-3a7a-4e49-b688-adbc5bcc83fe {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.416613] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1723.425102] env[63273]: DEBUG nova.compute.provider_tree [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1723.433861] env[63273]: DEBUG nova.network.neutron [-] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1723.435669] env[63273]: DEBUG nova.scheduler.client.report [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1723.441991] env[63273]: INFO nova.compute.manager [-] [instance: c4c785e1-274c-4713-98e1-9d7e5e8249a0] Took 0.04 seconds to deallocate network for instance. [ 1723.451328] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.436s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1723.452149] env[63273]: ERROR nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Failed to build and run instance: nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Traceback (most recent call last): [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] result = getattr(controller, method)(*args, **kwargs) [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._get(image_id) [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] resp, body = self.http_client.get(url, headers=header) [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.request(url, 'GET', **kwargs) [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._handle_response(resp) [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise exc.from_response(resp, resp.content) [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] During handling of the above exception, another exception occurred: [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Traceback (most recent call last): [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self.driver.spawn(context, instance, image_meta, [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._fetch_image_if_missing(context, vi) [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] image_fetch(context, vi, tmp_image_ds_loc) [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] images.fetch_image( [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] metadata = IMAGE_API.get(context, image_ref) [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1723.452149] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return session.show(context, image_id, [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] _reraise_translated_image_exception(image_id) [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise new_exc.with_traceback(exc_trace) [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] result = getattr(controller, method)(*args, **kwargs) [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._get(image_id) [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] resp, body = self.http_client.get(url, headers=header) [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.request(url, 'GET', **kwargs) [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._handle_response(resp) [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise exc.from_response(resp, resp.content) [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1723.453504] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.453504] env[63273]: DEBUG nova.compute.utils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1723.454589] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.037s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1723.455337] env[63273]: INFO nova.compute.claims [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1723.459086] env[63273]: DEBUG nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Build of instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 was re-scheduled: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1723.459086] env[63273]: DEBUG nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1723.459086] env[63273]: DEBUG nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1723.459086] env[63273]: DEBUG nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1723.459453] env[63273]: DEBUG nova.network.neutron [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1723.553915] env[63273]: DEBUG oslo_concurrency.lockutils [None req-69a9cfe5-81ef-424d-955c-cc7b59978dbe tempest-AttachVolumeShelveTestJSON-309788696 tempest-AttachVolumeShelveTestJSON-309788696-project-member] Lock "c4c785e1-274c-4713-98e1-9d7e5e8249a0" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.240s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1723.592853] env[63273]: DEBUG neutronclient.v2_0.client [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=63273) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1723.594124] env[63273]: ERROR nova.compute.manager [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Failed to deallocate networks: nova.exception.Unauthorized: Not authorized. [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Traceback (most recent call last): [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] result = getattr(controller, method)(*args, **kwargs) [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._get(image_id) [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] resp, body = self.http_client.get(url, headers=header) [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.request(url, 'GET', **kwargs) [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._handle_response(resp) [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise exc.from_response(resp, resp.content) [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] During handling of the above exception, another exception occurred: [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Traceback (most recent call last): [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self.driver.spawn(context, instance, image_meta, [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._fetch_image_if_missing(context, vi) [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] image_fetch(context, vi, tmp_image_ds_loc) [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] images.fetch_image( [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] metadata = IMAGE_API.get(context, image_ref) [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1723.594124] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return session.show(context, image_id, [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] _reraise_translated_image_exception(image_id) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise new_exc.with_traceback(exc_trace) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] result = getattr(controller, method)(*args, **kwargs) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._get(image_id) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] resp, body = self.http_client.get(url, headers=header) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.request(url, 'GET', **kwargs) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self._handle_response(resp) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise exc.from_response(resp, resp.content) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] nova.exception.ImageNotAuthorized: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] During handling of the above exception, another exception occurred: [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Traceback (most recent call last): [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 2448, in _do_build_and_run_instance [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._build_and_run_instance(context, instance, image, [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 2740, in _build_and_run_instance [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise exception.RescheduledException( [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] nova.exception.RescheduledException: Build of instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 was re-scheduled: Not authorized for image 443b5631-8941-44d6-8565-68279d3c04e8. [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] During handling of the above exception, another exception occurred: [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Traceback (most recent call last): [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] exception_handler_v20(status_code, error_body) [ 1723.595440] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise client_exc(message=error_message, [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Neutron server returns request_ids: ['req-3351284f-f050-4b81-b3f5-98022a4c7084'] [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] During handling of the above exception, another exception occurred: [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Traceback (most recent call last): [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 3037, in _cleanup_allocated_networks [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._deallocate_network(context, instance, requested_networks) [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self.network_api.deallocate_for_instance( [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] data = neutron.list_ports(**search_opts) [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.list('ports', self.ports_path, retrieve_all, [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] for r in self._pagination(collection, path, **params): [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] res = self.get(path, params=params) [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.retry_request("GET", action, body=body, [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.do_request(method, action, body=body, [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._handle_fault_response(status_code, replybody, resp) [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 204, in wrapper [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise exception.Unauthorized() [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] nova.exception.Unauthorized: Not authorized. [ 1723.597267] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.664195] env[63273]: INFO nova.scheduler.client.report [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Deleted allocations for instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 [ 1723.689583] env[63273]: DEBUG oslo_concurrency.lockutils [None req-3e5cbec1-3c62-47c9-af7f-2b1ca4f9200b tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 570.623s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1723.690695] env[63273]: DEBUG oslo_concurrency.lockutils [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 374.324s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1723.690916] env[63273]: DEBUG oslo_concurrency.lockutils [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Acquiring lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1723.691138] env[63273]: DEBUG oslo_concurrency.lockutils [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1723.691314] env[63273]: DEBUG oslo_concurrency.lockutils [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1723.696701] env[63273]: INFO nova.compute.manager [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Terminating instance [ 1723.699338] env[63273]: DEBUG nova.compute.manager [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1723.699338] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1723.699623] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e16a5d44-512e-4a05-b6fd-715d2b26fa2e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.709333] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1393024-374a-447a-89f1-c2c73752fbbb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.721391] env[63273]: DEBUG nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1723.724317] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3eff6f3-aee7-4671-8691-b5adbf432121 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.732638] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e93ee10f-3169-408d-aa5c-137adfa3f973 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.748415] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5780b2d7-f5b8-47be-8e0e-3d881f15cc90 could not be found. [ 1723.748415] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1723.748415] env[63273]: INFO nova.compute.manager [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1723.748415] env[63273]: DEBUG oslo.service.loopingcall [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1723.771691] env[63273]: DEBUG nova.compute.manager [-] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1723.771812] env[63273]: DEBUG nova.network.neutron [-] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1723.777112] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02d81924-8b63-4cbf-a0b0-76333e51b630 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.788430] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-849dc830-dc36-4ec6-967f-6a32c0381fe5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1723.796326] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1723.804864] env[63273]: DEBUG nova.compute.provider_tree [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1723.817603] env[63273]: DEBUG nova.scheduler.client.report [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1723.835983] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.382s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1723.837455] env[63273]: DEBUG nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1723.840868] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.043s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1723.840868] env[63273]: INFO nova.compute.claims [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1723.875556] env[63273]: DEBUG nova.compute.utils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1723.876815] env[63273]: DEBUG nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1723.876989] env[63273]: DEBUG nova.network.neutron [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1723.891298] env[63273]: DEBUG nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1723.903914] env[63273]: DEBUG neutronclient.v2_0.client [-] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=63273) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1723.904609] env[63273]: ERROR nova.network.neutron [-] Neutron client was not able to generate a valid admin token, please verify Neutron admin credential located in nova.conf: neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall [-] Dynamic interval looping call 'oslo_service.loopingcall.RetryDecorator.__call__.._func' failed: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall exception_handler_v20(status_code, error_body) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall raise client_exc(message=error_message, [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall Neutron server returns request_ids: ['req-3396ad3e-281a-42fb-93db-31495f5ba75d'] [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall During handling of the above exception, another exception occurred: [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall result = func(*self.args, **self.kw) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall result = f(*args, **kwargs) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall self._deallocate_network( [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall self.network_api.deallocate_for_instance( [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall data = neutron.list_ports(**search_opts) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall return self.list('ports', self.ports_path, retrieve_all, [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall for r in self._pagination(collection, path, **params): [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall res = self.get(path, params=params) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall return self.retry_request("GET", action, body=body, [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall return self.do_request(method, action, body=body, [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall self._handle_fault_response(status_code, replybody, resp) [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1723.904759] env[63273]: ERROR oslo.service.loopingcall [ 1723.906361] env[63273]: ERROR nova.compute.manager [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Failed to deallocate network for instance. Error: Networking client is experiencing an unauthorized exception.: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1723.937316] env[63273]: ERROR nova.compute.manager [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Setting instance vm_state to ERROR: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Traceback (most recent call last): [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] exception_handler_v20(status_code, error_body) [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise client_exc(message=error_message, [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Neutron server returns request_ids: ['req-3396ad3e-281a-42fb-93db-31495f5ba75d'] [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] During handling of the above exception, another exception occurred: [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Traceback (most recent call last): [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 3333, in do_terminate_instance [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._delete_instance(context, instance, bdms) [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 3268, in _delete_instance [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._shutdown_instance(context, instance, bdms) [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 3162, in _shutdown_instance [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._try_deallocate_network(context, instance, requested_networks) [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 3076, in _try_deallocate_network [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] with excutils.save_and_reraise_exception(): [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self.force_reraise() [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise self.value [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 3074, in _try_deallocate_network [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] _deallocate_network_with_retries() [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return evt.wait() [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] result = hub.switch() [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.greenlet.switch() [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] result = func(*self.args, **self.kw) [ 1723.937316] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] result = f(*args, **kwargs) [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._deallocate_network( [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self.network_api.deallocate_for_instance( [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] data = neutron.list_ports(**search_opts) [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.list('ports', self.ports_path, retrieve_all, [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] for r in self._pagination(collection, path, **params): [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] res = self.get(path, params=params) [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.retry_request("GET", action, body=body, [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] return self.do_request(method, action, body=body, [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] ret = obj(*args, **kwargs) [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] self._handle_fault_response(status_code, replybody, resp) [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1723.938904] env[63273]: ERROR nova.compute.manager [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] [ 1723.950882] env[63273]: DEBUG nova.policy [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '611c75633a8443e389b4736d21f6ca58', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ff660b4621c14dc48d0e1fad4c290dcc', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1723.962804] env[63273]: DEBUG nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1723.966634] env[63273]: DEBUG oslo_concurrency.lockutils [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Lock "5780b2d7-f5b8-47be-8e0e-3d881f15cc90" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.276s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1723.990052] env[63273]: DEBUG nova.virt.hardware [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1723.990303] env[63273]: DEBUG nova.virt.hardware [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1723.990485] env[63273]: DEBUG nova.virt.hardware [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1723.990684] env[63273]: DEBUG nova.virt.hardware [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1723.990834] env[63273]: DEBUG nova.virt.hardware [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1723.990982] env[63273]: DEBUG nova.virt.hardware [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1723.991268] env[63273]: DEBUG nova.virt.hardware [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1723.991462] env[63273]: DEBUG nova.virt.hardware [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1723.991644] env[63273]: DEBUG nova.virt.hardware [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1723.991811] env[63273]: DEBUG nova.virt.hardware [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1723.991984] env[63273]: DEBUG nova.virt.hardware [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1723.992873] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e279468-5008-4b3c-9f5c-a0add0d455e3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1724.006354] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e079175-c465-4c44-af96-ce1de82cf3c4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1724.030085] env[63273]: INFO nova.compute.manager [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] [instance: 5780b2d7-f5b8-47be-8e0e-3d881f15cc90] Successfully reverted task state from None on failure for instance. [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server [None req-855b1e1e-8234-43ee-9be9-5d6a124db816 tempest-ServersTestMultiNic-1506881135 tempest-ServersTestMultiNic-1506881135-project-member] Exception during message handling: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server exception_handler_v20(status_code, error_body) [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server raise client_exc(message=error_message, [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server Neutron server returns request_ids: ['req-3396ad3e-281a-42fb-93db-31495f5ba75d'] [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 65, in wrapped [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw) [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 166, in decorated_function [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 157, in decorated_function [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/utils.py", line 1453, in decorated_function [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 213, in decorated_function [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 203, in decorated_function [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3345, in terminate_instance [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server do_terminate_instance(instance, bdms) [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py", line 412, in inner [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server return f(*args, **kwargs) [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3340, in do_terminate_instance [ 1724.034039] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3333, in do_terminate_instance [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server self._delete_instance(context, instance, bdms) [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3268, in _delete_instance [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server self._shutdown_instance(context, instance, bdms) [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3162, in _shutdown_instance [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server self._try_deallocate_network(context, instance, requested_networks) [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3076, in _try_deallocate_network [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server raise self.value [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3074, in _try_deallocate_network [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server _deallocate_network_with_retries() [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server return evt.wait() [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server result = hub.switch() [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server return self.greenlet.switch() [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server result = func(*self.args, **self.kw) [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server result = f(*args, **kwargs) [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3063, in _deallocate_network_with_retries [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server self._deallocate_network( [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 2283, in _deallocate_network [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server self.network_api.deallocate_for_instance( [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server data = neutron.list_ports(**search_opts) [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server return self.list('ports', self.ports_path, retrieve_all, [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server for r in self._pagination(collection, path, **params): [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server res = self.get(path, params=params) [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server return self.retry_request("GET", action, body=body, [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server return self.do_request(method, action, body=body, [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1724.035780] env[63273]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1724.037385] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1724.037385] env[63273]: ERROR oslo_messaging.rpc.server self._handle_fault_response(status_code, replybody, resp) [ 1724.037385] env[63273]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1724.037385] env[63273]: ERROR oslo_messaging.rpc.server raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1724.037385] env[63273]: ERROR oslo_messaging.rpc.server nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1724.037385] env[63273]: ERROR oslo_messaging.rpc.server [ 1724.050430] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44a1d326-4b17-4477-9545-dbbfebe7e18e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1724.058327] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25ae600c-5e63-48cf-8af8-74ca70632ade {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1724.090234] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4990c683-97e4-41c7-9db9-a7f8a58782f1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1724.098016] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d49840c-b12d-4a90-a906-e7c8e6eadfae {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1724.111483] env[63273]: DEBUG nova.compute.provider_tree [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1724.120048] env[63273]: DEBUG nova.scheduler.client.report [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1724.135740] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.297s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1724.136331] env[63273]: DEBUG nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1724.171944] env[63273]: DEBUG nova.compute.utils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1724.173457] env[63273]: DEBUG nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1724.173672] env[63273]: DEBUG nova.network.neutron [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1724.183840] env[63273]: DEBUG nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1724.245821] env[63273]: DEBUG nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1724.249533] env[63273]: DEBUG nova.policy [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7648b8a75224424f8226b574da373a87', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '495ae26325184812a744fc95da67038b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1724.268047] env[63273]: DEBUG nova.virt.hardware [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1724.268299] env[63273]: DEBUG nova.virt.hardware [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1724.268458] env[63273]: DEBUG nova.virt.hardware [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1724.268644] env[63273]: DEBUG nova.virt.hardware [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1724.268791] env[63273]: DEBUG nova.virt.hardware [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1724.268935] env[63273]: DEBUG nova.virt.hardware [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1724.269194] env[63273]: DEBUG nova.virt.hardware [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1724.269332] env[63273]: DEBUG nova.virt.hardware [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1724.269500] env[63273]: DEBUG nova.virt.hardware [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1724.269660] env[63273]: DEBUG nova.virt.hardware [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1724.269835] env[63273]: DEBUG nova.virt.hardware [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1724.270924] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a9f5b77-4087-4a49-94b9-c770ad9c6e4a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1724.283023] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-437dfcf2-885b-4cf5-a704-cabe2770b780 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1724.348732] env[63273]: DEBUG nova.network.neutron [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Successfully created port: e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1724.889746] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1724.978466] env[63273]: DEBUG nova.network.neutron [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Successfully created port: 559ba11f-8ca5-45a5-b51f-e1bcd3d55d32 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1725.420440] env[63273]: DEBUG nova.compute.manager [req-d5e74142-77ef-471a-935c-458093acddc1 req-69cbc899-0b31-4c9c-920c-c43e9742ce90 service nova] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Received event network-vif-plugged-e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1725.420697] env[63273]: DEBUG oslo_concurrency.lockutils [req-d5e74142-77ef-471a-935c-458093acddc1 req-69cbc899-0b31-4c9c-920c-c43e9742ce90 service nova] Acquiring lock "2054144d-f068-4d52-b913-617b30d2d1c0-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1725.420890] env[63273]: DEBUG oslo_concurrency.lockutils [req-d5e74142-77ef-471a-935c-458093acddc1 req-69cbc899-0b31-4c9c-920c-c43e9742ce90 service nova] Lock "2054144d-f068-4d52-b913-617b30d2d1c0-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1725.421096] env[63273]: DEBUG oslo_concurrency.lockutils [req-d5e74142-77ef-471a-935c-458093acddc1 req-69cbc899-0b31-4c9c-920c-c43e9742ce90 service nova] Lock "2054144d-f068-4d52-b913-617b30d2d1c0-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1725.421327] env[63273]: DEBUG nova.compute.manager [req-d5e74142-77ef-471a-935c-458093acddc1 req-69cbc899-0b31-4c9c-920c-c43e9742ce90 service nova] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] No waiting events found dispatching network-vif-plugged-e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1725.421393] env[63273]: WARNING nova.compute.manager [req-d5e74142-77ef-471a-935c-458093acddc1 req-69cbc899-0b31-4c9c-920c-c43e9742ce90 service nova] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Received unexpected event network-vif-plugged-e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e for instance with vm_state building and task_state spawning. [ 1725.541888] env[63273]: DEBUG nova.network.neutron [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Successfully updated port: e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1725.554568] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "refresh_cache-2054144d-f068-4d52-b913-617b30d2d1c0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1725.554568] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquired lock "refresh_cache-2054144d-f068-4d52-b913-617b30d2d1c0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1725.554695] env[63273]: DEBUG nova.network.neutron [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1725.670694] env[63273]: DEBUG nova.network.neutron [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1725.891924] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1725.892122] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11307}} [ 1725.906356] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] There are 0 instances to clean {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11316}} [ 1726.040956] env[63273]: DEBUG nova.network.neutron [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Updating instance_info_cache with network_info: [{"id": "e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e", "address": "fa:16:3e:a0:a1:b3", "network": {"id": "f04a4851-a302-4dcf-97cb-b3c86b22cd29", "bridge": "br-int", "label": "tempest-SecurityGroupsTestJSON-1730702358-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff660b4621c14dc48d0e1fad4c290dcc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "62f28d75-4e6a-4ae5-b8b3-d0652ea26d08", "external-id": "nsx-vlan-transportzone-801", "segmentation_id": 801, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape5b8c78c-dd", "ovs_interfaceid": "e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1726.052935] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Releasing lock "refresh_cache-2054144d-f068-4d52-b913-617b30d2d1c0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1726.053245] env[63273]: DEBUG nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Instance network_info: |[{"id": "e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e", "address": "fa:16:3e:a0:a1:b3", "network": {"id": "f04a4851-a302-4dcf-97cb-b3c86b22cd29", "bridge": "br-int", "label": "tempest-SecurityGroupsTestJSON-1730702358-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff660b4621c14dc48d0e1fad4c290dcc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "62f28d75-4e6a-4ae5-b8b3-d0652ea26d08", "external-id": "nsx-vlan-transportzone-801", "segmentation_id": 801, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape5b8c78c-dd", "ovs_interfaceid": "e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1726.053689] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:a0:a1:b3', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '62f28d75-4e6a-4ae5-b8b3-d0652ea26d08', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1726.061636] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Creating folder: Project (ff660b4621c14dc48d0e1fad4c290dcc). Parent ref: group-v986930. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1726.062345] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-bb63a565-beac-4987-87ac-df94480cf488 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1726.074490] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Created folder: Project (ff660b4621c14dc48d0e1fad4c290dcc) in parent group-v986930. [ 1726.075327] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Creating folder: Instances. Parent ref: group-v987030. {{(pid=63273) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1726.075327] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-e5f45e0f-3a4e-4367-ba30-b5725f63ad2e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1726.085045] env[63273]: INFO nova.virt.vmwareapi.vm_util [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Created folder: Instances in parent group-v987030. [ 1726.085361] env[63273]: DEBUG oslo.service.loopingcall [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1726.085557] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1726.085770] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-b57948b4-cec5-440e-9c5a-6669eeebee4a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1726.109957] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1726.109957] env[63273]: value = "task-5072165" [ 1726.109957] env[63273]: _type = "Task" [ 1726.109957] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1726.118631] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072165, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1726.161900] env[63273]: DEBUG nova.network.neutron [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Successfully updated port: 559ba11f-8ca5-45a5-b51f-e1bcd3d55d32 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1726.173639] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "refresh_cache-7262f23a-8ee4-4468-8e49-2e9297352dc9" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1726.173801] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "refresh_cache-7262f23a-8ee4-4468-8e49-2e9297352dc9" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1726.174094] env[63273]: DEBUG nova.network.neutron [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1726.242464] env[63273]: DEBUG nova.network.neutron [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1726.443830] env[63273]: DEBUG nova.network.neutron [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Updating instance_info_cache with network_info: [{"id": "559ba11f-8ca5-45a5-b51f-e1bcd3d55d32", "address": "fa:16:3e:82:be:ac", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap559ba11f-8c", "ovs_interfaceid": "559ba11f-8ca5-45a5-b51f-e1bcd3d55d32", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1726.458479] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "refresh_cache-7262f23a-8ee4-4468-8e49-2e9297352dc9" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1726.458789] env[63273]: DEBUG nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Instance network_info: |[{"id": "559ba11f-8ca5-45a5-b51f-e1bcd3d55d32", "address": "fa:16:3e:82:be:ac", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap559ba11f-8c", "ovs_interfaceid": "559ba11f-8ca5-45a5-b51f-e1bcd3d55d32", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1726.459240] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:82:be:ac', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '69e1a5c1-4ae7-409b-8de7-d401684ef60d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '559ba11f-8ca5-45a5-b51f-e1bcd3d55d32', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1726.466855] env[63273]: DEBUG oslo.service.loopingcall [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1726.467387] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1726.467624] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-e6aa792b-1dfe-4e6e-8556-38dc34443be1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1726.488377] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1726.488377] env[63273]: value = "task-5072166" [ 1726.488377] env[63273]: _type = "Task" [ 1726.488377] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1726.496932] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072166, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1726.620883] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072165, 'name': CreateVM_Task, 'duration_secs': 0.318735} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1726.621082] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1726.621776] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1726.621943] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1726.622286] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1726.622619] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f8d92572-7b39-49f8-81d2-06898bc0d987 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1726.627857] env[63273]: DEBUG oslo_vmware.api [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for the task: (returnval){ [ 1726.627857] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52df0062-405c-0184-2ce8-0f380bb42efa" [ 1726.627857] env[63273]: _type = "Task" [ 1726.627857] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1726.636655] env[63273]: DEBUG oslo_vmware.api [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52df0062-405c-0184-2ce8-0f380bb42efa, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1726.999334] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072166, 'name': CreateVM_Task, 'duration_secs': 0.287191} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1726.999960] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1727.000266] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1727.138896] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1727.139237] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1727.139559] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1727.139721] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1727.140038] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1727.140305] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-57a559ed-fe0f-4d1c-950f-0b24752ff915 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1727.146071] env[63273]: DEBUG oslo_vmware.api [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 1727.146071] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52731a4c-5f43-3470-aadf-4fe0f9277d97" [ 1727.146071] env[63273]: _type = "Task" [ 1727.146071] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1727.155304] env[63273]: DEBUG oslo_vmware.api [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52731a4c-5f43-3470-aadf-4fe0f9277d97, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1727.449563] env[63273]: DEBUG nova.compute.manager [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Received event network-changed-e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1727.449853] env[63273]: DEBUG nova.compute.manager [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Refreshing instance network info cache due to event network-changed-e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1727.449909] env[63273]: DEBUG oslo_concurrency.lockutils [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] Acquiring lock "refresh_cache-2054144d-f068-4d52-b913-617b30d2d1c0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1727.450038] env[63273]: DEBUG oslo_concurrency.lockutils [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] Acquired lock "refresh_cache-2054144d-f068-4d52-b913-617b30d2d1c0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1727.450202] env[63273]: DEBUG nova.network.neutron [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Refreshing network info cache for port e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1727.661189] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1727.661637] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1727.662787] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1727.721635] env[63273]: DEBUG nova.network.neutron [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Updated VIF entry in instance network info cache for port e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1727.722009] env[63273]: DEBUG nova.network.neutron [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Updating instance_info_cache with network_info: [{"id": "e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e", "address": "fa:16:3e:a0:a1:b3", "network": {"id": "f04a4851-a302-4dcf-97cb-b3c86b22cd29", "bridge": "br-int", "label": "tempest-SecurityGroupsTestJSON-1730702358-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff660b4621c14dc48d0e1fad4c290dcc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "62f28d75-4e6a-4ae5-b8b3-d0652ea26d08", "external-id": "nsx-vlan-transportzone-801", "segmentation_id": 801, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape5b8c78c-dd", "ovs_interfaceid": "e5b8c78c-dd03-4c2d-958d-a6ccb2a3058e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1727.731933] env[63273]: DEBUG oslo_concurrency.lockutils [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] Releasing lock "refresh_cache-2054144d-f068-4d52-b913-617b30d2d1c0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1727.732190] env[63273]: DEBUG nova.compute.manager [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Received event network-vif-plugged-559ba11f-8ca5-45a5-b51f-e1bcd3d55d32 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1727.732380] env[63273]: DEBUG oslo_concurrency.lockutils [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] Acquiring lock "7262f23a-8ee4-4468-8e49-2e9297352dc9-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1727.732619] env[63273]: DEBUG oslo_concurrency.lockutils [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] Lock "7262f23a-8ee4-4468-8e49-2e9297352dc9-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1727.732790] env[63273]: DEBUG oslo_concurrency.lockutils [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] Lock "7262f23a-8ee4-4468-8e49-2e9297352dc9-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1727.732962] env[63273]: DEBUG nova.compute.manager [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] No waiting events found dispatching network-vif-plugged-559ba11f-8ca5-45a5-b51f-e1bcd3d55d32 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1727.733129] env[63273]: WARNING nova.compute.manager [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Received unexpected event network-vif-plugged-559ba11f-8ca5-45a5-b51f-e1bcd3d55d32 for instance with vm_state building and task_state spawning. [ 1727.733287] env[63273]: DEBUG nova.compute.manager [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Received event network-changed-559ba11f-8ca5-45a5-b51f-e1bcd3d55d32 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1727.733437] env[63273]: DEBUG nova.compute.manager [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Refreshing instance network info cache due to event network-changed-559ba11f-8ca5-45a5-b51f-e1bcd3d55d32. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1727.733663] env[63273]: DEBUG oslo_concurrency.lockutils [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] Acquiring lock "refresh_cache-7262f23a-8ee4-4468-8e49-2e9297352dc9" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1727.733812] env[63273]: DEBUG oslo_concurrency.lockutils [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] Acquired lock "refresh_cache-7262f23a-8ee4-4468-8e49-2e9297352dc9" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1727.733966] env[63273]: DEBUG nova.network.neutron [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Refreshing network info cache for port 559ba11f-8ca5-45a5-b51f-e1bcd3d55d32 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1728.062465] env[63273]: DEBUG nova.network.neutron [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Updated VIF entry in instance network info cache for port 559ba11f-8ca5-45a5-b51f-e1bcd3d55d32. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1728.062919] env[63273]: DEBUG nova.network.neutron [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Updating instance_info_cache with network_info: [{"id": "559ba11f-8ca5-45a5-b51f-e1bcd3d55d32", "address": "fa:16:3e:82:be:ac", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap559ba11f-8c", "ovs_interfaceid": "559ba11f-8ca5-45a5-b51f-e1bcd3d55d32", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1728.071748] env[63273]: DEBUG oslo_concurrency.lockutils [req-d42e98f3-1a7d-4a3f-8c0a-124acb8b3c90 req-0111dd7b-612a-4d80-a703-cb427c66fb81 service nova] Releasing lock "refresh_cache-7262f23a-8ee4-4468-8e49-2e9297352dc9" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1748.701249] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquiring lock "92426c26-52fa-4978-a382-3684ec018bc5" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1770.191453] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_power_states {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1770.214247] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Getting list of instances from cluster (obj){ [ 1770.214247] env[63273]: value = "domain-c8" [ 1770.214247] env[63273]: _type = "ClusterComputeResource" [ 1770.214247] env[63273]: } {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 1770.215583] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93ab6611-d7db-4f7a-97a1-1aaf17ad0bb7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1770.234739] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Got total of 10 instances {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 1770.234979] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid fa31b240-8bad-48ba-8339-155dc6acb265 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1770.235148] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 1da174da-b4e6-437c-a538-53fc78b4282f {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1770.235366] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid efe849d5-9df6-4813-a23b-c805e7eb2456 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1770.235455] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1770.235616] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 61bd6d66-590f-4512-afbf-9abf4a308749 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1770.235797] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1770.236082] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 92426c26-52fa-4978-a382-3684ec018bc5 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1770.236082] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 0afc321d-108c-4c2e-81a7-664091127047 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1770.236342] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 2054144d-f068-4d52-b913-617b30d2d1c0 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1770.236342] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 7262f23a-8ee4-4468-8e49-2e9297352dc9 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 1770.236760] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "fa31b240-8bad-48ba-8339-155dc6acb265" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1770.238729] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "1da174da-b4e6-437c-a538-53fc78b4282f" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1770.238729] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "efe849d5-9df6-4813-a23b-c805e7eb2456" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1770.238729] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1770.238729] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "61bd6d66-590f-4512-afbf-9abf4a308749" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1770.238729] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1770.238729] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "92426c26-52fa-4978-a382-3684ec018bc5" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1770.238729] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "0afc321d-108c-4c2e-81a7-664091127047" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1770.238729] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "2054144d-f068-4d52-b913-617b30d2d1c0" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1770.238729] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "7262f23a-8ee4-4468-8e49-2e9297352dc9" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1770.430286] env[63273]: WARNING oslo_vmware.rw_handles [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1770.430286] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1770.430286] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1770.430286] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1770.430286] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1770.430286] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1770.430286] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1770.430286] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1770.430286] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1770.430286] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1770.430286] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1770.430286] env[63273]: ERROR oslo_vmware.rw_handles [ 1770.430801] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/2021caf8-c1ac-432c-a550-9f8966e20e6d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1770.432850] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1770.433147] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Copying Virtual Disk [datastore1] vmware_temp/2021caf8-c1ac-432c-a550-9f8966e20e6d/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/2021caf8-c1ac-432c-a550-9f8966e20e6d/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1770.433511] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-4ea34a8d-e3bf-4c8d-8913-c3a8f2705be3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1770.442481] env[63273]: DEBUG oslo_vmware.api [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 1770.442481] env[63273]: value = "task-5072167" [ 1770.442481] env[63273]: _type = "Task" [ 1770.442481] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1770.453188] env[63273]: DEBUG oslo_vmware.api [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': task-5072167, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1770.953059] env[63273]: DEBUG oslo_vmware.exceptions [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1770.953370] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1770.953965] env[63273]: ERROR nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1770.953965] env[63273]: Faults: ['InvalidArgument'] [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Traceback (most recent call last): [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] yield resources [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] self.driver.spawn(context, instance, image_meta, [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] self._fetch_image_if_missing(context, vi) [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] image_cache(vi, tmp_image_ds_loc) [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] vm_util.copy_virtual_disk( [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] session._wait_for_task(vmdk_copy_task) [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] return self.wait_for_task(task_ref) [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] return evt.wait() [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] result = hub.switch() [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] return self.greenlet.switch() [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] self.f(*self.args, **self.kw) [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] raise exceptions.translate_fault(task_info.error) [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Faults: ['InvalidArgument'] [ 1770.953965] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] [ 1770.955385] env[63273]: INFO nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Terminating instance [ 1770.955989] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1770.956221] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1770.956488] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0da96fac-dc40-492a-9340-4930bd18dbcc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1770.958980] env[63273]: DEBUG nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1770.959190] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1770.959955] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d9608df-45fe-41ee-965c-beb2be322539 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1770.967917] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1770.968189] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-8b82110e-6564-4cad-9f81-1de8239641fd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1770.970729] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1770.970941] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1770.971954] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-023703c7-27d5-4a3f-8f8e-2f6e1efabc08 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1770.977913] env[63273]: DEBUG oslo_vmware.api [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 1770.977913] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5212e47b-ae78-9109-5cf9-64b751d0638a" [ 1770.977913] env[63273]: _type = "Task" [ 1770.977913] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1770.986082] env[63273]: DEBUG oslo_vmware.api [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5212e47b-ae78-9109-5cf9-64b751d0638a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1771.037979] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1771.038234] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1771.038441] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Deleting the datastore file [datastore1] fa31b240-8bad-48ba-8339-155dc6acb265 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1771.038681] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-6de0d23e-75bb-426c-8bb9-178a7cb7d198 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1771.045951] env[63273]: DEBUG oslo_vmware.api [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 1771.045951] env[63273]: value = "task-5072169" [ 1771.045951] env[63273]: _type = "Task" [ 1771.045951] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1771.057811] env[63273]: DEBUG oslo_vmware.api [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': task-5072169, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1771.488647] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1771.489050] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating directory with path [datastore1] vmware_temp/4fe63de3-90b3-4a55-8216-1dc0ab5c06dd/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1771.489245] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-2b68af67-e282-4d2f-bc35-35c8a9ef04ac {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1771.501011] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created directory with path [datastore1] vmware_temp/4fe63de3-90b3-4a55-8216-1dc0ab5c06dd/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1771.501250] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Fetch image to [datastore1] vmware_temp/4fe63de3-90b3-4a55-8216-1dc0ab5c06dd/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1771.501471] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/4fe63de3-90b3-4a55-8216-1dc0ab5c06dd/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1771.502250] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2cd582e8-19b9-4a1f-8a72-5700775ebbd4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1771.509618] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a5b1cb8-61b5-4782-aad4-493a93bf380e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1771.519630] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41495f11-f2e9-4105-a17c-b4bd7333f910 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1771.554717] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-babb4a15-bff7-4248-b17b-f32aa8357cca {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1771.563569] env[63273]: DEBUG oslo_vmware.api [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': task-5072169, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.073587} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1771.564096] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1771.564289] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1771.564526] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1771.564721] env[63273]: INFO nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Took 0.61 seconds to destroy the instance on the hypervisor. [ 1771.566336] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-78c88fd1-273b-4b83-a767-c0d523b2371c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1771.568254] env[63273]: DEBUG nova.compute.claims [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1771.568426] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1771.568658] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1771.590892] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1771.648561] env[63273]: DEBUG oslo_vmware.rw_handles [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4fe63de3-90b3-4a55-8216-1dc0ab5c06dd/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1771.714239] env[63273]: DEBUG oslo_vmware.rw_handles [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1771.714458] env[63273]: DEBUG oslo_vmware.rw_handles [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4fe63de3-90b3-4a55-8216-1dc0ab5c06dd/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1771.717977] env[63273]: DEBUG nova.scheduler.client.report [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Refreshing inventories for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 1771.734039] env[63273]: DEBUG nova.scheduler.client.report [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Updating ProviderTree inventory for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 1771.734269] env[63273]: DEBUG nova.compute.provider_tree [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Updating inventory in ProviderTree for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1771.746456] env[63273]: DEBUG nova.scheduler.client.report [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Refreshing aggregate associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, aggregates: None {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 1771.765778] env[63273]: DEBUG nova.scheduler.client.report [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Refreshing trait associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NODE,COMPUTE_IMAGE_TYPE_ISO {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 1771.898602] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff6a709e-3388-4ee7-89af-e99070f8b6f3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1771.907546] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7b47974-76fd-4a49-8ec7-c810768c2be6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1771.936768] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-925cf181-104d-4801-b88c-dc955983a4f3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1771.944688] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f3fb1c7-3f58-4ac9-aa7c-0fb4b38e5067 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1771.957309] env[63273]: DEBUG nova.compute.provider_tree [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1771.968311] env[63273]: DEBUG nova.scheduler.client.report [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1771.983726] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.415s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1771.984991] env[63273]: ERROR nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1771.984991] env[63273]: Faults: ['InvalidArgument'] [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Traceback (most recent call last): [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] self.driver.spawn(context, instance, image_meta, [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] self._fetch_image_if_missing(context, vi) [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] image_cache(vi, tmp_image_ds_loc) [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] vm_util.copy_virtual_disk( [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] session._wait_for_task(vmdk_copy_task) [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] return self.wait_for_task(task_ref) [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] return evt.wait() [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] result = hub.switch() [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] return self.greenlet.switch() [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] self.f(*self.args, **self.kw) [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] raise exceptions.translate_fault(task_info.error) [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Faults: ['InvalidArgument'] [ 1771.984991] env[63273]: ERROR nova.compute.manager [instance: fa31b240-8bad-48ba-8339-155dc6acb265] [ 1771.984991] env[63273]: DEBUG nova.compute.utils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1771.986558] env[63273]: DEBUG nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Build of instance fa31b240-8bad-48ba-8339-155dc6acb265 was re-scheduled: A specified parameter was not correct: fileType [ 1771.986558] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1771.986933] env[63273]: DEBUG nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1771.987120] env[63273]: DEBUG nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1771.987297] env[63273]: DEBUG nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1771.987458] env[63273]: DEBUG nova.network.neutron [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1772.416693] env[63273]: DEBUG nova.network.neutron [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1772.427563] env[63273]: INFO nova.compute.manager [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Took 0.44 seconds to deallocate network for instance. [ 1772.539809] env[63273]: INFO nova.scheduler.client.report [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Deleted allocations for instance fa31b240-8bad-48ba-8339-155dc6acb265 [ 1772.564356] env[63273]: DEBUG oslo_concurrency.lockutils [None req-883cb688-8747-4ab4-9dc1-33374091ea68 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fa31b240-8bad-48ba-8339-155dc6acb265" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 566.252s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1772.564665] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fa31b240-8bad-48ba-8339-155dc6acb265" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 370.022s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1772.564900] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "fa31b240-8bad-48ba-8339-155dc6acb265-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1772.565145] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fa31b240-8bad-48ba-8339-155dc6acb265-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1772.565322] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fa31b240-8bad-48ba-8339-155dc6acb265-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1772.567641] env[63273]: INFO nova.compute.manager [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Terminating instance [ 1772.570094] env[63273]: DEBUG nova.compute.manager [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1772.570268] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1772.570548] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-dc545e22-4303-4be2-99ed-fb341f590891 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1772.582547] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-395e7538-7ed7-4711-bad8-d5680bb999af {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1772.618496] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fa31b240-8bad-48ba-8339-155dc6acb265 could not be found. [ 1772.618736] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1772.618922] env[63273]: INFO nova.compute.manager [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1772.619206] env[63273]: DEBUG oslo.service.loopingcall [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1772.619540] env[63273]: DEBUG nova.compute.manager [-] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1772.619641] env[63273]: DEBUG nova.network.neutron [-] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1772.646390] env[63273]: DEBUG nova.network.neutron [-] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1772.656028] env[63273]: INFO nova.compute.manager [-] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] Took 0.04 seconds to deallocate network for instance. [ 1772.756807] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1e4efc65-9eab-4a93-be0d-c15cf78b0d84 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fa31b240-8bad-48ba-8339-155dc6acb265" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.192s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1772.757754] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "fa31b240-8bad-48ba-8339-155dc6acb265" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 2.521s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1772.757934] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fa31b240-8bad-48ba-8339-155dc6acb265] During sync_power_state the instance has a pending task (deleting). Skip. [ 1772.758126] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "fa31b240-8bad-48ba-8339-155dc6acb265" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1772.938833] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1774.891508] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1775.891992] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1775.905059] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1775.905279] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1775.905455] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1775.905609] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1775.906796] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d425c6d5-8229-4d4f-b1b9-67efcc0e05d3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1775.916635] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4de2f798-b75e-4734-8c94-9d1389cf7247 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1775.932852] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3026f435-891c-47a6-95d6-f6ca8e4de527 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1775.940203] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe8e4bc2-3514-47d5-b1df-b70a2f5dbd25 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1775.970129] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180519MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1775.970308] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1775.970497] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1776.074719] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 1da174da-b4e6-437c-a538-53fc78b4282f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1776.074881] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance efe849d5-9df6-4813-a23b-c805e7eb2456 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1776.075018] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1776.075265] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 61bd6d66-590f-4512-afbf-9abf4a308749 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1776.075467] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1776.075606] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92426c26-52fa-4978-a382-3684ec018bc5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1776.075731] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0afc321d-108c-4c2e-81a7-664091127047 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1776.075852] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2054144d-f068-4d52-b913-617b30d2d1c0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1776.075971] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1776.076190] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 9 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1776.076349] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=100GB used_disk=9GB total_vcpus=48 used_vcpus=9 pci_stats=[] stats={'failed_builds': '79', 'num_instances': '9', 'num_vm_building': '9', 'num_task_deleting': '6', 'num_os_type_None': '9', 'num_proj_495ae26325184812a744fc95da67038b': '2', 'io_workload': '9', 'num_proj_72f5eacfc9004694a5107a00edabbdd6': '1', 'num_proj_d277c5d52dd44fbdab7560c6fb34cf31': '1', 'num_proj_a5420f4321c84264957a02c02f47649f': '1', 'num_proj_5683e7f13d7540c1bda6b8e43f1428d0': '1', 'num_proj_846931e260f34a22a2894c7cb88722fa': '1', 'num_task_spawning': '3', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1', 'num_proj_ff660b4621c14dc48d0e1fad4c290dcc': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1776.197986] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd863e5a-e0dd-48c3-9d9e-1ec7adcc2e09 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1776.206244] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6ad47ef-bc9a-4b95-b15d-435b05826241 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1776.237299] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f55be84-0b12-44d4-a777-ab2ffbc914b4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1776.245571] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4aa4bdb0-b615-43b6-a321-b588d95cf4f9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1777.005924] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1777.015260] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1777.029496] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1777.029709] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.059s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1779.030058] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1779.030393] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1779.030393] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1779.049519] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1779.049681] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1779.049799] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1779.049925] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1779.050058] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1779.050172] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1779.050292] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1779.050405] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1779.050519] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1779.050633] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1779.051141] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1779.892161] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1780.892492] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1783.891241] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1783.891632] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1785.887167] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1795.609277] env[63273]: DEBUG oslo_concurrency.lockutils [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "0afc321d-108c-4c2e-81a7-664091127047" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1810.358430] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "886b6d46-98a1-4954-9113-0aadfd507d29" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1810.358802] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "886b6d46-98a1-4954-9113-0aadfd507d29" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1810.373272] env[63273]: DEBUG nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1810.430808] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1810.430808] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1810.432519] env[63273]: INFO nova.compute.claims [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1810.607903] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-303d2585-c3b8-4591-bd9b-b2eff6660ec2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.615819] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1631b565-cf9b-4e74-9da8-db7e7c69209e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.645127] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40633420-dd28-4234-9e45-d0e0ef5ab750 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.653594] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2bec068-2c94-4014-9c34-542dd3a25503 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.667499] env[63273]: DEBUG nova.compute.provider_tree [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1810.678498] env[63273]: DEBUG nova.scheduler.client.report [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1810.693398] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.262s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1810.693918] env[63273]: DEBUG nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1810.728906] env[63273]: DEBUG nova.compute.utils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1810.730505] env[63273]: DEBUG nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1810.730702] env[63273]: DEBUG nova.network.neutron [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1810.741303] env[63273]: DEBUG nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1810.792546] env[63273]: DEBUG nova.policy [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b9cfeeeafe8448ea85b67a6cd1c54e46', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5683e7f13d7540c1bda6b8e43f1428d0', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1810.818185] env[63273]: DEBUG nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1810.845216] env[63273]: DEBUG nova.virt.hardware [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1810.845464] env[63273]: DEBUG nova.virt.hardware [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1810.845625] env[63273]: DEBUG nova.virt.hardware [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1810.845802] env[63273]: DEBUG nova.virt.hardware [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1810.845958] env[63273]: DEBUG nova.virt.hardware [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1810.846118] env[63273]: DEBUG nova.virt.hardware [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1810.846327] env[63273]: DEBUG nova.virt.hardware [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1810.846487] env[63273]: DEBUG nova.virt.hardware [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1810.846650] env[63273]: DEBUG nova.virt.hardware [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1810.846810] env[63273]: DEBUG nova.virt.hardware [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1810.846978] env[63273]: DEBUG nova.virt.hardware [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1810.847839] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8dd7ced0-23bc-4cc8-82a7-94a994f64574 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.856639] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f782c26-424d-4bba-a628-f7004c26573e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1811.128812] env[63273]: DEBUG nova.network.neutron [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Successfully created port: bc7908c7-23a5-432d-b3c4-247ec911e98e {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1811.797142] env[63273]: DEBUG nova.compute.manager [req-c9e8fd82-717d-42ad-aff4-9aeac5ad369e req-4e6e28b0-c17d-477c-a5f8-abeb41361bc7 service nova] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Received event network-vif-plugged-bc7908c7-23a5-432d-b3c4-247ec911e98e {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1811.797142] env[63273]: DEBUG oslo_concurrency.lockutils [req-c9e8fd82-717d-42ad-aff4-9aeac5ad369e req-4e6e28b0-c17d-477c-a5f8-abeb41361bc7 service nova] Acquiring lock "886b6d46-98a1-4954-9113-0aadfd507d29-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1811.797142] env[63273]: DEBUG oslo_concurrency.lockutils [req-c9e8fd82-717d-42ad-aff4-9aeac5ad369e req-4e6e28b0-c17d-477c-a5f8-abeb41361bc7 service nova] Lock "886b6d46-98a1-4954-9113-0aadfd507d29-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1811.797142] env[63273]: DEBUG oslo_concurrency.lockutils [req-c9e8fd82-717d-42ad-aff4-9aeac5ad369e req-4e6e28b0-c17d-477c-a5f8-abeb41361bc7 service nova] Lock "886b6d46-98a1-4954-9113-0aadfd507d29-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1811.797142] env[63273]: DEBUG nova.compute.manager [req-c9e8fd82-717d-42ad-aff4-9aeac5ad369e req-4e6e28b0-c17d-477c-a5f8-abeb41361bc7 service nova] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] No waiting events found dispatching network-vif-plugged-bc7908c7-23a5-432d-b3c4-247ec911e98e {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1811.797142] env[63273]: WARNING nova.compute.manager [req-c9e8fd82-717d-42ad-aff4-9aeac5ad369e req-4e6e28b0-c17d-477c-a5f8-abeb41361bc7 service nova] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Received unexpected event network-vif-plugged-bc7908c7-23a5-432d-b3c4-247ec911e98e for instance with vm_state building and task_state spawning. [ 1811.882138] env[63273]: DEBUG nova.network.neutron [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Successfully updated port: bc7908c7-23a5-432d-b3c4-247ec911e98e {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1811.907529] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "refresh_cache-886b6d46-98a1-4954-9113-0aadfd507d29" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1811.907729] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquired lock "refresh_cache-886b6d46-98a1-4954-9113-0aadfd507d29" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1811.907939] env[63273]: DEBUG nova.network.neutron [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1811.955228] env[63273]: DEBUG nova.network.neutron [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1812.136448] env[63273]: DEBUG nova.network.neutron [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Updating instance_info_cache with network_info: [{"id": "bc7908c7-23a5-432d-b3c4-247ec911e98e", "address": "fa:16:3e:71:29:ba", "network": {"id": "a7f8a09b-70fc-4a63-89a4-585b22da5db6", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-896645902-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5683e7f13d7540c1bda6b8e43f1428d0", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "bc9714ff-7109-4ea1-9435-b2b3fbdb9e81", "external-id": "nsx-vlan-transportzone-887", "segmentation_id": 887, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbc7908c7-23", "ovs_interfaceid": "bc7908c7-23a5-432d-b3c4-247ec911e98e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1812.150647] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Releasing lock "refresh_cache-886b6d46-98a1-4954-9113-0aadfd507d29" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1812.150964] env[63273]: DEBUG nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Instance network_info: |[{"id": "bc7908c7-23a5-432d-b3c4-247ec911e98e", "address": "fa:16:3e:71:29:ba", "network": {"id": "a7f8a09b-70fc-4a63-89a4-585b22da5db6", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-896645902-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5683e7f13d7540c1bda6b8e43f1428d0", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "bc9714ff-7109-4ea1-9435-b2b3fbdb9e81", "external-id": "nsx-vlan-transportzone-887", "segmentation_id": 887, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbc7908c7-23", "ovs_interfaceid": "bc7908c7-23a5-432d-b3c4-247ec911e98e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1812.151437] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:71:29:ba', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'bc9714ff-7109-4ea1-9435-b2b3fbdb9e81', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'bc7908c7-23a5-432d-b3c4-247ec911e98e', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1812.159068] env[63273]: DEBUG oslo.service.loopingcall [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1812.159570] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1812.159816] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-e01f7944-d331-4b03-9089-547dee11d570 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1812.179781] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1812.179781] env[63273]: value = "task-5072170" [ 1812.179781] env[63273]: _type = "Task" [ 1812.179781] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1812.188944] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072170, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1812.690711] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072170, 'name': CreateVM_Task, 'duration_secs': 0.291235} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1812.690913] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1812.699072] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1812.699072] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1812.699325] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1812.699598] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-fa521772-21af-4679-a596-9fecbc24c9db {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1812.704550] env[63273]: DEBUG oslo_vmware.api [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for the task: (returnval){ [ 1812.704550] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52b25381-751b-2a3b-01a8-28811edd4531" [ 1812.704550] env[63273]: _type = "Task" [ 1812.704550] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1812.712856] env[63273]: DEBUG oslo_vmware.api [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52b25381-751b-2a3b-01a8-28811edd4531, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1813.214708] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1813.215140] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1813.215224] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1813.824678] env[63273]: DEBUG nova.compute.manager [req-20c8caf2-3fc9-47a6-806c-18c7a688d430 req-022f1990-5bf9-4910-98d9-800bc97b1d0e service nova] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Received event network-changed-bc7908c7-23a5-432d-b3c4-247ec911e98e {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1813.824883] env[63273]: DEBUG nova.compute.manager [req-20c8caf2-3fc9-47a6-806c-18c7a688d430 req-022f1990-5bf9-4910-98d9-800bc97b1d0e service nova] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Refreshing instance network info cache due to event network-changed-bc7908c7-23a5-432d-b3c4-247ec911e98e. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1813.825109] env[63273]: DEBUG oslo_concurrency.lockutils [req-20c8caf2-3fc9-47a6-806c-18c7a688d430 req-022f1990-5bf9-4910-98d9-800bc97b1d0e service nova] Acquiring lock "refresh_cache-886b6d46-98a1-4954-9113-0aadfd507d29" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1813.825255] env[63273]: DEBUG oslo_concurrency.lockutils [req-20c8caf2-3fc9-47a6-806c-18c7a688d430 req-022f1990-5bf9-4910-98d9-800bc97b1d0e service nova] Acquired lock "refresh_cache-886b6d46-98a1-4954-9113-0aadfd507d29" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1813.825415] env[63273]: DEBUG nova.network.neutron [req-20c8caf2-3fc9-47a6-806c-18c7a688d430 req-022f1990-5bf9-4910-98d9-800bc97b1d0e service nova] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Refreshing network info cache for port bc7908c7-23a5-432d-b3c4-247ec911e98e {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1814.345329] env[63273]: DEBUG nova.network.neutron [req-20c8caf2-3fc9-47a6-806c-18c7a688d430 req-022f1990-5bf9-4910-98d9-800bc97b1d0e service nova] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Updated VIF entry in instance network info cache for port bc7908c7-23a5-432d-b3c4-247ec911e98e. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1814.345711] env[63273]: DEBUG nova.network.neutron [req-20c8caf2-3fc9-47a6-806c-18c7a688d430 req-022f1990-5bf9-4910-98d9-800bc97b1d0e service nova] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Updating instance_info_cache with network_info: [{"id": "bc7908c7-23a5-432d-b3c4-247ec911e98e", "address": "fa:16:3e:71:29:ba", "network": {"id": "a7f8a09b-70fc-4a63-89a4-585b22da5db6", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-896645902-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5683e7f13d7540c1bda6b8e43f1428d0", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "bc9714ff-7109-4ea1-9435-b2b3fbdb9e81", "external-id": "nsx-vlan-transportzone-887", "segmentation_id": 887, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbc7908c7-23", "ovs_interfaceid": "bc7908c7-23a5-432d-b3c4-247ec911e98e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1814.355613] env[63273]: DEBUG oslo_concurrency.lockutils [req-20c8caf2-3fc9-47a6-806c-18c7a688d430 req-022f1990-5bf9-4910-98d9-800bc97b1d0e service nova] Releasing lock "refresh_cache-886b6d46-98a1-4954-9113-0aadfd507d29" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1820.430916] env[63273]: WARNING oslo_vmware.rw_handles [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1820.430916] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1820.430916] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1820.430916] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1820.430916] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1820.430916] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1820.430916] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1820.430916] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1820.430916] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1820.430916] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1820.430916] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1820.430916] env[63273]: ERROR oslo_vmware.rw_handles [ 1820.430916] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/4fe63de3-90b3-4a55-8216-1dc0ab5c06dd/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1820.433686] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1820.433686] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Copying Virtual Disk [datastore1] vmware_temp/4fe63de3-90b3-4a55-8216-1dc0ab5c06dd/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/4fe63de3-90b3-4a55-8216-1dc0ab5c06dd/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1820.433936] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-478e7674-58c8-4f16-bb82-444d81e78bda {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.443208] env[63273]: DEBUG oslo_vmware.api [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 1820.443208] env[63273]: value = "task-5072171" [ 1820.443208] env[63273]: _type = "Task" [ 1820.443208] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1820.452693] env[63273]: DEBUG oslo_vmware.api [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072171, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1820.953781] env[63273]: DEBUG oslo_vmware.exceptions [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1820.954053] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1820.954669] env[63273]: ERROR nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1820.954669] env[63273]: Faults: ['InvalidArgument'] [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Traceback (most recent call last): [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] yield resources [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] self.driver.spawn(context, instance, image_meta, [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] self._fetch_image_if_missing(context, vi) [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] image_cache(vi, tmp_image_ds_loc) [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] vm_util.copy_virtual_disk( [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] session._wait_for_task(vmdk_copy_task) [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] return self.wait_for_task(task_ref) [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] return evt.wait() [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] result = hub.switch() [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] return self.greenlet.switch() [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] self.f(*self.args, **self.kw) [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] raise exceptions.translate_fault(task_info.error) [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Faults: ['InvalidArgument'] [ 1820.954669] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] [ 1820.955905] env[63273]: INFO nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Terminating instance [ 1820.956612] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1820.956823] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1820.957082] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-fe3b171a-f9c5-4e11-9175-938f98f7a41a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.959622] env[63273]: DEBUG nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1820.959816] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1820.960557] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e5cc2ebd-c9cb-400c-af3b-67a596fd850d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.967687] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1820.967926] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-bcec1cea-12d7-4d60-b834-061a5a7e90db {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.970199] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1820.970408] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1820.971355] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-060bc92e-b5d4-4def-b424-f7ff7de35ef3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.976447] env[63273]: DEBUG oslo_vmware.api [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Waiting for the task: (returnval){ [ 1820.976447] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52168030-4253-5d3b-57a4-357e3034b622" [ 1820.976447] env[63273]: _type = "Task" [ 1820.976447] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1820.988586] env[63273]: DEBUG oslo_vmware.api [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52168030-4253-5d3b-57a4-357e3034b622, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1821.042354] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1821.042575] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1821.042794] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleting the datastore file [datastore1] 1da174da-b4e6-437c-a538-53fc78b4282f {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1821.043033] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-8ac3880e-2ebe-4a52-880d-fc7c3b123a9a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.049820] env[63273]: DEBUG oslo_vmware.api [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 1821.049820] env[63273]: value = "task-5072173" [ 1821.049820] env[63273]: _type = "Task" [ 1821.049820] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1821.058770] env[63273]: DEBUG oslo_vmware.api [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072173, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1821.487768] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1821.488189] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Creating directory with path [datastore1] vmware_temp/8353975a-2a7f-4649-acfb-5a0da8c35472/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1821.488328] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-238c9ed2-c3d3-479f-b158-82a55a2b3ebb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.501304] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Created directory with path [datastore1] vmware_temp/8353975a-2a7f-4649-acfb-5a0da8c35472/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1821.501608] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Fetch image to [datastore1] vmware_temp/8353975a-2a7f-4649-acfb-5a0da8c35472/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1821.501739] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/8353975a-2a7f-4649-acfb-5a0da8c35472/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1821.502559] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7e45d7a-580e-4eb8-84fa-5717d0efdc03 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.510136] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f6677e8-3208-4233-8b21-a1fac4c3637d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.519705] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8651950d-0f3f-49fb-b299-59cb5b43f41b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.556595] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84cea9b3-c388-4b73-b7f3-f4c40177c198 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.566039] env[63273]: DEBUG oslo_vmware.api [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072173, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.071188} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1821.566570] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1821.566797] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1821.567042] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1821.567156] env[63273]: INFO nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Took 0.61 seconds to destroy the instance on the hypervisor. [ 1821.568718] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-74259b1c-a440-4cb2-9705-3405d7837401 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.570841] env[63273]: DEBUG nova.compute.claims [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1821.571024] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1821.571244] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1821.600193] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1821.656264] env[63273]: DEBUG oslo_vmware.rw_handles [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/8353975a-2a7f-4649-acfb-5a0da8c35472/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1821.721414] env[63273]: DEBUG oslo_vmware.rw_handles [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1821.721597] env[63273]: DEBUG oslo_vmware.rw_handles [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/8353975a-2a7f-4649-acfb-5a0da8c35472/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1821.818020] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f6b7826-59bd-4cdd-9ac2-a6fcd1ca6941 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.826067] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-702b03d0-81b1-4365-8eae-107ac242bbe7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.857250] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1ceffb0-99dc-41c8-b6ad-1555bb250e97 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.865350] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1067b2b8-0115-49b6-bc62-692f3220780c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.881308] env[63273]: DEBUG nova.compute.provider_tree [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1821.890376] env[63273]: DEBUG nova.scheduler.client.report [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1821.907601] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.336s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1821.907916] env[63273]: ERROR nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1821.907916] env[63273]: Faults: ['InvalidArgument'] [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Traceback (most recent call last): [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] self.driver.spawn(context, instance, image_meta, [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] self._fetch_image_if_missing(context, vi) [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] image_cache(vi, tmp_image_ds_loc) [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] vm_util.copy_virtual_disk( [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] session._wait_for_task(vmdk_copy_task) [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] return self.wait_for_task(task_ref) [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] return evt.wait() [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] result = hub.switch() [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] return self.greenlet.switch() [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] self.f(*self.args, **self.kw) [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] raise exceptions.translate_fault(task_info.error) [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Faults: ['InvalidArgument'] [ 1821.907916] env[63273]: ERROR nova.compute.manager [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] [ 1821.908906] env[63273]: DEBUG nova.compute.utils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1821.910580] env[63273]: DEBUG nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Build of instance 1da174da-b4e6-437c-a538-53fc78b4282f was re-scheduled: A specified parameter was not correct: fileType [ 1821.910580] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1821.910962] env[63273]: DEBUG nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1821.911147] env[63273]: DEBUG nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1821.911418] env[63273]: DEBUG nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1821.911682] env[63273]: DEBUG nova.network.neutron [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1822.180981] env[63273]: DEBUG nova.network.neutron [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1822.193995] env[63273]: INFO nova.compute.manager [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Took 0.28 seconds to deallocate network for instance. [ 1822.302697] env[63273]: INFO nova.scheduler.client.report [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleted allocations for instance 1da174da-b4e6-437c-a538-53fc78b4282f [ 1822.327148] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7ab654ec-9ab3-475c-b7e0-c7ebcc7d0594 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "1da174da-b4e6-437c-a538-53fc78b4282f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 530.062s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1822.327148] env[63273]: DEBUG oslo_concurrency.lockutils [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "1da174da-b4e6-437c-a538-53fc78b4282f" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 334.682s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1822.327148] env[63273]: DEBUG oslo_concurrency.lockutils [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "1da174da-b4e6-437c-a538-53fc78b4282f-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.327360] env[63273]: DEBUG oslo_concurrency.lockutils [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "1da174da-b4e6-437c-a538-53fc78b4282f-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1822.327414] env[63273]: DEBUG oslo_concurrency.lockutils [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "1da174da-b4e6-437c-a538-53fc78b4282f-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1822.329924] env[63273]: INFO nova.compute.manager [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Terminating instance [ 1822.331770] env[63273]: DEBUG nova.compute.manager [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1822.331969] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1822.332480] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c00f0d20-c65d-459d-bfc7-641ac36a9d8b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1822.341937] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb914ae8-4bf9-4efe-9b43-e227668d3213 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1822.374970] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1da174da-b4e6-437c-a538-53fc78b4282f could not be found. [ 1822.375194] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1822.375380] env[63273]: INFO nova.compute.manager [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1822.375632] env[63273]: DEBUG oslo.service.loopingcall [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1822.375878] env[63273]: DEBUG nova.compute.manager [-] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1822.375970] env[63273]: DEBUG nova.network.neutron [-] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1822.402197] env[63273]: DEBUG nova.network.neutron [-] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1822.411605] env[63273]: INFO nova.compute.manager [-] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] Took 0.04 seconds to deallocate network for instance. [ 1822.502578] env[63273]: DEBUG oslo_concurrency.lockutils [None req-227de463-1230-496a-8700-39952a8febe7 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "1da174da-b4e6-437c-a538-53fc78b4282f" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.175s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1822.504021] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "1da174da-b4e6-437c-a538-53fc78b4282f" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 52.267s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1822.504329] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 1da174da-b4e6-437c-a538-53fc78b4282f] During sync_power_state the instance has a pending task (deleting). Skip. [ 1822.504557] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "1da174da-b4e6-437c-a538-53fc78b4282f" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1830.885995] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "2054144d-f068-4d52-b913-617b30d2d1c0" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1834.891467] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1836.892102] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1837.887759] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1837.910368] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1837.910813] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1837.921665] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1837.921889] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1837.922071] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1837.922231] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1837.923317] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93c056fb-2bda-4c2a-a90c-c7d8027ce80f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1837.932152] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27d5546b-6c73-46e7-882b-ce2ca7b47ecb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1837.945920] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee6befdc-c4a2-4f76-97d2-526b6c05ff27 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1837.952109] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce3aff73-4a14-440e-b466-93db998de39c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1837.979553] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180545MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1837.979712] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1837.979896] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1838.050812] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance efe849d5-9df6-4813-a23b-c805e7eb2456 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1838.050971] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1838.051124] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 61bd6d66-590f-4512-afbf-9abf4a308749 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1838.051229] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1838.051343] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92426c26-52fa-4978-a382-3684ec018bc5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1838.051459] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0afc321d-108c-4c2e-81a7-664091127047 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1838.051568] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2054144d-f068-4d52-b913-617b30d2d1c0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1838.051682] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1838.051791] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 886b6d46-98a1-4954-9113-0aadfd507d29 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1838.051972] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 9 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1838.052132] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=100GB used_disk=9GB total_vcpus=48 used_vcpus=9 pci_stats=[] stats={'failed_builds': '80', 'num_instances': '9', 'num_vm_building': '9', 'num_task_deleting': '7', 'num_os_type_None': '9', 'num_proj_72f5eacfc9004694a5107a00edabbdd6': '1', 'io_workload': '9', 'num_proj_d277c5d52dd44fbdab7560c6fb34cf31': '1', 'num_proj_a5420f4321c84264957a02c02f47649f': '1', 'num_proj_5683e7f13d7540c1bda6b8e43f1428d0': '2', 'num_proj_846931e260f34a22a2894c7cb88722fa': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1', 'num_proj_ff660b4621c14dc48d0e1fad4c290dcc': '1', 'num_task_spawning': '2', 'num_proj_495ae26325184812a744fc95da67038b': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1838.169778] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4208eb2f-7cae-46d1-a98e-527dee33c641 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1838.177614] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d8c33a0-3c49-46bd-b5c4-a9c57c0e6e26 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1838.206806] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd01e06b-0f4d-4d71-985f-639252ee46c8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1838.214608] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf6f25ca-0c26-4f31-8dd5-0578a90f87c9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1838.227728] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1838.236314] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1838.251301] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1838.251478] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.272s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1840.233429] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1840.892398] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1840.892609] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1840.892682] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1840.912339] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1840.912514] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1840.912616] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1840.912743] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1840.912863] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1840.912980] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1840.913163] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1840.913228] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1840.913343] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1840.913461] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1842.892058] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1843.892417] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1843.892839] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1847.887127] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1871.150050] env[63273]: WARNING oslo_vmware.rw_handles [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1871.150050] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1871.150050] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1871.150050] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1871.150050] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1871.150050] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1871.150050] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1871.150050] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1871.150050] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1871.150050] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1871.150050] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1871.150050] env[63273]: ERROR oslo_vmware.rw_handles [ 1871.150948] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/8353975a-2a7f-4649-acfb-5a0da8c35472/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1871.152390] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1871.152631] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Copying Virtual Disk [datastore1] vmware_temp/8353975a-2a7f-4649-acfb-5a0da8c35472/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/8353975a-2a7f-4649-acfb-5a0da8c35472/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1871.152917] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-d0b9ff3e-4dbd-4056-8448-e14941f9e343 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1871.162334] env[63273]: DEBUG oslo_vmware.api [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Waiting for the task: (returnval){ [ 1871.162334] env[63273]: value = "task-5072174" [ 1871.162334] env[63273]: _type = "Task" [ 1871.162334] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1871.170990] env[63273]: DEBUG oslo_vmware.api [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Task: {'id': task-5072174, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1871.673871] env[63273]: DEBUG oslo_vmware.exceptions [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1871.674152] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1871.674711] env[63273]: ERROR nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1871.674711] env[63273]: Faults: ['InvalidArgument'] [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Traceback (most recent call last): [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] yield resources [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] self.driver.spawn(context, instance, image_meta, [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] self._fetch_image_if_missing(context, vi) [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] image_cache(vi, tmp_image_ds_loc) [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] vm_util.copy_virtual_disk( [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] session._wait_for_task(vmdk_copy_task) [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] return self.wait_for_task(task_ref) [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] return evt.wait() [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] result = hub.switch() [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] return self.greenlet.switch() [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] self.f(*self.args, **self.kw) [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] raise exceptions.translate_fault(task_info.error) [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Faults: ['InvalidArgument'] [ 1871.674711] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] [ 1871.675984] env[63273]: INFO nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Terminating instance [ 1871.676714] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1871.676971] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1871.677222] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-73edb6e5-aedc-4b3c-921c-962f6763dc62 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1871.679474] env[63273]: DEBUG nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1871.679670] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1871.680398] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3764130b-209a-4a62-b980-e889f9bbf72a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1871.687850] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1871.688078] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-ae026117-46cb-4c6e-8ffb-634b8bb7f827 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1871.690266] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1871.690436] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1871.691368] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-12a3e4bd-c071-48dc-98f8-6c9bf17aaf11 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1871.696247] env[63273]: DEBUG oslo_vmware.api [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Waiting for the task: (returnval){ [ 1871.696247] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]526a6eeb-e556-b0a6-10a4-ea8b4edd883a" [ 1871.696247] env[63273]: _type = "Task" [ 1871.696247] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1871.704524] env[63273]: DEBUG oslo_vmware.api [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]526a6eeb-e556-b0a6-10a4-ea8b4edd883a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1871.756851] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1871.757195] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1871.757399] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Deleting the datastore file [datastore1] efe849d5-9df6-4813-a23b-c805e7eb2456 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1871.757683] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-802ff735-2df9-497c-9f1c-f7c5c45187d5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1871.766018] env[63273]: DEBUG oslo_vmware.api [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Waiting for the task: (returnval){ [ 1871.766018] env[63273]: value = "task-5072176" [ 1871.766018] env[63273]: _type = "Task" [ 1871.766018] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1871.774032] env[63273]: DEBUG oslo_vmware.api [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Task: {'id': task-5072176, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1872.206814] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1872.207190] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Creating directory with path [datastore1] vmware_temp/71e2d269-9a79-4b2f-89db-2b183087aed2/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1872.207309] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-eb87db18-aa6d-4e6f-938e-c9db11c977d5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.219762] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Created directory with path [datastore1] vmware_temp/71e2d269-9a79-4b2f-89db-2b183087aed2/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1872.220035] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Fetch image to [datastore1] vmware_temp/71e2d269-9a79-4b2f-89db-2b183087aed2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1872.220178] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/71e2d269-9a79-4b2f-89db-2b183087aed2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1872.220898] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3119c36f-c899-4671-809b-5a7e2e177024 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.227895] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9aca5aa-58a5-43ce-ad65-1d47bec26239 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.237192] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ec0d014-ba81-44d4-b3f1-ce5ad8193608 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.270959] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebc6668a-35ff-4d79-a836-55b3f4ba9484 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.278997] env[63273]: DEBUG oslo_vmware.api [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Task: {'id': task-5072176, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.076803} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1872.280419] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1872.280610] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1872.280803] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1872.280977] env[63273]: INFO nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1872.283128] env[63273]: DEBUG nova.compute.claims [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1872.283336] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1872.283495] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1872.286053] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-f302a285-35c6-4629-8ef2-c0b510420a84 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.309850] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1872.367278] env[63273]: DEBUG oslo_vmware.rw_handles [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/71e2d269-9a79-4b2f-89db-2b183087aed2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1872.426690] env[63273]: DEBUG oslo_vmware.rw_handles [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1872.426877] env[63273]: DEBUG oslo_vmware.rw_handles [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/71e2d269-9a79-4b2f-89db-2b183087aed2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1872.502444] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b550217-3169-4b5d-811c-9a9368768e2d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.510456] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17bee650-cb2e-4655-b994-87bd79e40616 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.541960] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6bfbcbb-ba11-48de-b048-190be5ca06ad {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.550344] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-750e2547-12f7-4fd2-85d0-40be56f3a2fa {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.563842] env[63273]: DEBUG nova.compute.provider_tree [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1872.572837] env[63273]: DEBUG nova.scheduler.client.report [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1872.586692] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.303s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1872.587913] env[63273]: ERROR nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1872.587913] env[63273]: Faults: ['InvalidArgument'] [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Traceback (most recent call last): [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] self.driver.spawn(context, instance, image_meta, [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] self._fetch_image_if_missing(context, vi) [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] image_cache(vi, tmp_image_ds_loc) [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] vm_util.copy_virtual_disk( [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] session._wait_for_task(vmdk_copy_task) [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] return self.wait_for_task(task_ref) [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] return evt.wait() [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] result = hub.switch() [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] return self.greenlet.switch() [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] self.f(*self.args, **self.kw) [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] raise exceptions.translate_fault(task_info.error) [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Faults: ['InvalidArgument'] [ 1872.587913] env[63273]: ERROR nova.compute.manager [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] [ 1872.589080] env[63273]: DEBUG nova.compute.utils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1872.589522] env[63273]: DEBUG nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Build of instance efe849d5-9df6-4813-a23b-c805e7eb2456 was re-scheduled: A specified parameter was not correct: fileType [ 1872.589522] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1872.589898] env[63273]: DEBUG nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1872.590085] env[63273]: DEBUG nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1872.590261] env[63273]: DEBUG nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1872.590427] env[63273]: DEBUG nova.network.neutron [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1872.947236] env[63273]: DEBUG nova.network.neutron [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1872.960178] env[63273]: INFO nova.compute.manager [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Took 0.37 seconds to deallocate network for instance. [ 1873.054940] env[63273]: INFO nova.scheduler.client.report [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Deleted allocations for instance efe849d5-9df6-4813-a23b-c805e7eb2456 [ 1873.081019] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bed8957b-44fd-463e-bea4-cb8eaabb2f75 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Lock "efe849d5-9df6-4813-a23b-c805e7eb2456" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 527.134s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1873.081315] env[63273]: DEBUG oslo_concurrency.lockutils [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Lock "efe849d5-9df6-4813-a23b-c805e7eb2456" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 330.678s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1873.081541] env[63273]: DEBUG oslo_concurrency.lockutils [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Acquiring lock "efe849d5-9df6-4813-a23b-c805e7eb2456-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1873.081761] env[63273]: DEBUG oslo_concurrency.lockutils [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Lock "efe849d5-9df6-4813-a23b-c805e7eb2456-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1873.082103] env[63273]: DEBUG oslo_concurrency.lockutils [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Lock "efe849d5-9df6-4813-a23b-c805e7eb2456-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1873.084410] env[63273]: INFO nova.compute.manager [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Terminating instance [ 1873.086551] env[63273]: DEBUG nova.compute.manager [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1873.086741] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1873.087261] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7591561c-c3de-49da-b125-ae7bc3800c62 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1873.096949] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47275a01-8f94-4bd1-9395-d5e3ad308edc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1873.127609] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance efe849d5-9df6-4813-a23b-c805e7eb2456 could not be found. [ 1873.127824] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1873.128015] env[63273]: INFO nova.compute.manager [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1873.128281] env[63273]: DEBUG oslo.service.loopingcall [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1873.128521] env[63273]: DEBUG nova.compute.manager [-] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1873.128613] env[63273]: DEBUG nova.network.neutron [-] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1873.157809] env[63273]: DEBUG nova.network.neutron [-] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1873.166055] env[63273]: INFO nova.compute.manager [-] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] Took 0.04 seconds to deallocate network for instance. [ 1873.285977] env[63273]: DEBUG oslo_concurrency.lockutils [None req-22500d13-a257-4cca-96cf-4c735662a118 tempest-ServersNegativeTestJSON-943588607 tempest-ServersNegativeTestJSON-943588607-project-member] Lock "efe849d5-9df6-4813-a23b-c805e7eb2456" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.205s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1873.287029] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "efe849d5-9df6-4813-a23b-c805e7eb2456" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 103.050s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1873.287187] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: efe849d5-9df6-4813-a23b-c805e7eb2456] During sync_power_state the instance has a pending task (deleting). Skip. [ 1873.287366] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "efe849d5-9df6-4813-a23b-c805e7eb2456" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1880.077309] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "7262f23a-8ee4-4468-8e49-2e9297352dc9" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1895.892573] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1897.891973] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1897.891973] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1898.892063] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1898.904720] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1898.904993] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1898.905138] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1898.905534] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1898.906451] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71ea1579-39ba-4fc4-aa66-91e933a35e06 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1898.915328] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c07a1a36-4ddc-483d-b8f9-5c6a2da19350 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1898.929572] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51847798-ec76-41c6-bf85-f8d9d6a0b469 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1898.936865] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9825c62-2399-4681-b6d3-504adf7d4aa7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1898.965247] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180527MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1898.965398] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1898.965581] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1899.034830] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1899.035036] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 61bd6d66-590f-4512-afbf-9abf4a308749 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1899.035036] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1899.035211] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92426c26-52fa-4978-a382-3684ec018bc5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1899.035307] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0afc321d-108c-4c2e-81a7-664091127047 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1899.035405] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2054144d-f068-4d52-b913-617b30d2d1c0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1899.035509] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1899.035623] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 886b6d46-98a1-4954-9113-0aadfd507d29 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1899.035821] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 8 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1899.035973] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1536MB phys_disk=100GB used_disk=8GB total_vcpus=48 used_vcpus=8 pci_stats=[] stats={'failed_builds': '81', 'num_instances': '8', 'num_vm_building': '8', 'num_task_deleting': '7', 'num_os_type_None': '8', 'num_proj_d277c5d52dd44fbdab7560c6fb34cf31': '1', 'io_workload': '8', 'num_proj_a5420f4321c84264957a02c02f47649f': '1', 'num_proj_5683e7f13d7540c1bda6b8e43f1428d0': '2', 'num_proj_846931e260f34a22a2894c7cb88722fa': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1', 'num_proj_ff660b4621c14dc48d0e1fad4c290dcc': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_task_spawning': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1899.142945] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ceaaef5-1691-4aaa-a369-eb28d9c3ed58 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1899.151414] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8a2e460-fc44-4e89-8715-79b84379f16e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1899.183120] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03061615-8fba-499d-9f0a-074d573edd04 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1899.192085] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3ef9304-effb-42e6-820e-da0717e8402b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1899.206054] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1899.216061] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1899.233708] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1899.233920] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.268s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1900.234664] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1901.892759] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1901.893179] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1901.893179] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1901.911508] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1901.911671] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1901.911776] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1901.911902] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1901.912057] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1901.912230] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1901.912369] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1901.912492] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1901.912612] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1903.892679] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1904.891388] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1904.891388] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1908.887973] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1920.466303] env[63273]: WARNING oslo_vmware.rw_handles [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1920.466303] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1920.466303] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1920.466303] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1920.466303] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1920.466303] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1920.466303] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1920.466303] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1920.466303] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1920.466303] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1920.466303] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1920.466303] env[63273]: ERROR oslo_vmware.rw_handles [ 1920.467037] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/71e2d269-9a79-4b2f-89db-2b183087aed2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1920.468573] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1920.468823] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Copying Virtual Disk [datastore1] vmware_temp/71e2d269-9a79-4b2f-89db-2b183087aed2/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/71e2d269-9a79-4b2f-89db-2b183087aed2/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1920.469137] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-e1c944eb-f721-4a3f-9f62-40d1a048bc59 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1920.478987] env[63273]: DEBUG oslo_vmware.api [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Waiting for the task: (returnval){ [ 1920.478987] env[63273]: value = "task-5072177" [ 1920.478987] env[63273]: _type = "Task" [ 1920.478987] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1920.487588] env[63273]: DEBUG oslo_vmware.api [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Task: {'id': task-5072177, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1920.990081] env[63273]: DEBUG oslo_vmware.exceptions [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1920.990387] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1920.990893] env[63273]: ERROR nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1920.990893] env[63273]: Faults: ['InvalidArgument'] [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Traceback (most recent call last): [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] yield resources [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] self.driver.spawn(context, instance, image_meta, [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] self._fetch_image_if_missing(context, vi) [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] image_cache(vi, tmp_image_ds_loc) [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] vm_util.copy_virtual_disk( [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] session._wait_for_task(vmdk_copy_task) [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] return self.wait_for_task(task_ref) [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] return evt.wait() [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] result = hub.switch() [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] return self.greenlet.switch() [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] self.f(*self.args, **self.kw) [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] raise exceptions.translate_fault(task_info.error) [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Faults: ['InvalidArgument'] [ 1920.990893] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] [ 1920.992514] env[63273]: INFO nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Terminating instance [ 1920.993626] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1920.993626] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1920.993626] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-5138a7f1-83da-44f6-9da6-e5bdd7599e61 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1920.995593] env[63273]: DEBUG nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1920.995786] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1920.996532] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96a88963-d481-48bf-93d4-3221658dc8ed {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.004919] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1921.005205] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-a3f6c9d2-d226-4534-ac5a-153489575867 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.007716] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1921.007903] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1921.008957] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-620f3163-832a-4732-bc8d-b4ccf5fc48ce {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.014713] env[63273]: DEBUG oslo_vmware.api [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Waiting for the task: (returnval){ [ 1921.014713] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5278dfc8-45d7-8348-a1c7-62f533715f29" [ 1921.014713] env[63273]: _type = "Task" [ 1921.014713] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1921.024881] env[63273]: DEBUG oslo_vmware.api [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5278dfc8-45d7-8348-a1c7-62f533715f29, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1921.081240] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1921.081546] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1921.081762] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Deleting the datastore file [datastore1] d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1921.082191] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-16772c5c-114d-4ffc-90b0-7c26b887216f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.091791] env[63273]: DEBUG oslo_vmware.api [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Waiting for the task: (returnval){ [ 1921.091791] env[63273]: value = "task-5072179" [ 1921.091791] env[63273]: _type = "Task" [ 1921.091791] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1921.101233] env[63273]: DEBUG oslo_vmware.api [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Task: {'id': task-5072179, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1921.525767] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1921.526110] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Creating directory with path [datastore1] vmware_temp/6dde67f0-d4f8-401e-a682-18e35874b81b/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1921.526324] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0afd85c8-9a13-48dc-a5e1-042f584a4312 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.539798] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Created directory with path [datastore1] vmware_temp/6dde67f0-d4f8-401e-a682-18e35874b81b/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1921.540036] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Fetch image to [datastore1] vmware_temp/6dde67f0-d4f8-401e-a682-18e35874b81b/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1921.540184] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/6dde67f0-d4f8-401e-a682-18e35874b81b/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1921.540958] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4a8b5f9-3aaf-47ad-ae11-09508dc0bf76 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.548385] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-962b58fe-b49a-45d8-99d2-7d39007d935d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.558392] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa04278c-f472-4513-ae3e-50c5ed323ad5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.590139] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40db7639-8a92-4928-9a78-1fde63df75f6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.601025] env[63273]: DEBUG oslo_vmware.api [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Task: {'id': task-5072179, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.080433} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1921.603131] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1921.603363] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1921.603613] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1921.603759] env[63273]: INFO nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Took 0.61 seconds to destroy the instance on the hypervisor. [ 1921.605913] env[63273]: DEBUG nova.compute.claims [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1921.606097] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1921.606313] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1921.608984] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-187871c8-e124-444f-bf40-92f788482aaf {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.633868] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1921.696946] env[63273]: DEBUG oslo_vmware.rw_handles [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/6dde67f0-d4f8-401e-a682-18e35874b81b/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1921.765570] env[63273]: DEBUG oslo_vmware.rw_handles [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1921.765791] env[63273]: DEBUG oslo_vmware.rw_handles [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/6dde67f0-d4f8-401e-a682-18e35874b81b/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1921.824226] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12dbe918-a0df-4389-ba00-2af81d372a63 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.833742] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6434e6cb-63be-41ea-8942-43419b283a19 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.863146] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f8e4a45-f73b-4039-8327-256c69610017 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.871630] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2835b5c0-e34b-49e9-a2de-9476bdd8449e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1921.885971] env[63273]: DEBUG nova.compute.provider_tree [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1921.895344] env[63273]: DEBUG nova.scheduler.client.report [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1921.913397] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.307s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1921.913948] env[63273]: ERROR nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1921.913948] env[63273]: Faults: ['InvalidArgument'] [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Traceback (most recent call last): [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] self.driver.spawn(context, instance, image_meta, [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] self._fetch_image_if_missing(context, vi) [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] image_cache(vi, tmp_image_ds_loc) [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] vm_util.copy_virtual_disk( [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] session._wait_for_task(vmdk_copy_task) [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] return self.wait_for_task(task_ref) [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] return evt.wait() [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] result = hub.switch() [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] return self.greenlet.switch() [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] self.f(*self.args, **self.kw) [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] raise exceptions.translate_fault(task_info.error) [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Faults: ['InvalidArgument'] [ 1921.913948] env[63273]: ERROR nova.compute.manager [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] [ 1921.915088] env[63273]: DEBUG nova.compute.utils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1921.916348] env[63273]: DEBUG nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Build of instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f was re-scheduled: A specified parameter was not correct: fileType [ 1921.916348] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1921.916738] env[63273]: DEBUG nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1921.916904] env[63273]: DEBUG nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1921.917087] env[63273]: DEBUG nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1921.917254] env[63273]: DEBUG nova.network.neutron [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1922.231574] env[63273]: DEBUG nova.network.neutron [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1922.250063] env[63273]: INFO nova.compute.manager [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Took 0.33 seconds to deallocate network for instance. [ 1922.342258] env[63273]: INFO nova.scheduler.client.report [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Deleted allocations for instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f [ 1922.367671] env[63273]: DEBUG oslo_concurrency.lockutils [None req-7681f896-9f08-4a94-8dae-0ee477881a34 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 557.801s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1922.367861] env[63273]: DEBUG oslo_concurrency.lockutils [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 361.455s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1922.368110] env[63273]: DEBUG oslo_concurrency.lockutils [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Acquiring lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1922.368382] env[63273]: DEBUG oslo_concurrency.lockutils [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1922.368511] env[63273]: DEBUG oslo_concurrency.lockutils [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1922.370443] env[63273]: INFO nova.compute.manager [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Terminating instance [ 1922.372348] env[63273]: DEBUG nova.compute.manager [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1922.372582] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1922.373119] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9d889157-949e-4fe8-8c94-3002069dd9df {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1922.382845] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2f12d14-22b8-421d-a288-7ea647251a9e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1922.414195] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f could not be found. [ 1922.414505] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1922.414636] env[63273]: INFO nova.compute.manager [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1922.414888] env[63273]: DEBUG oslo.service.loopingcall [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1922.415132] env[63273]: DEBUG nova.compute.manager [-] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1922.415231] env[63273]: DEBUG nova.network.neutron [-] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1922.439111] env[63273]: DEBUG nova.network.neutron [-] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1922.446885] env[63273]: INFO nova.compute.manager [-] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] Took 0.03 seconds to deallocate network for instance. [ 1922.534249] env[63273]: DEBUG oslo_concurrency.lockutils [None req-abd20a69-0ebc-4770-b8d3-7bd033fecd37 tempest-ServerRescueTestJSON-16576647 tempest-ServerRescueTestJSON-16576647-project-member] Lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.166s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1922.535683] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 152.298s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1922.535683] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f] During sync_power_state the instance has a pending task (deleting). Skip. [ 1922.535683] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "d4ced5cf-c4ae-41ec-9087-72b8e3e8a46f" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1956.893488] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1957.891844] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1958.887640] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1958.908486] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1958.920482] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1958.920657] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1958.920826] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1958.920982] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1958.922096] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ba0159f-7870-440c-af2b-d8e0cd9e761b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1958.930759] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a869851-885b-45b9-a648-5f11f63163ab {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1958.945239] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe80d187-784f-43f3-8c94-6c607571cd3b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1958.952156] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-473f1e75-18b4-4558-8bfc-1129f2d7e9f2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1958.982752] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180527MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1958.982999] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1958.983271] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1959.049569] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 61bd6d66-590f-4512-afbf-9abf4a308749 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1959.049734] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1959.049867] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92426c26-52fa-4978-a382-3684ec018bc5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1959.049990] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0afc321d-108c-4c2e-81a7-664091127047 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1959.050129] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2054144d-f068-4d52-b913-617b30d2d1c0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1959.050246] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1959.050361] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 886b6d46-98a1-4954-9113-0aadfd507d29 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 1959.050550] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 7 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1959.050702] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1408MB phys_disk=100GB used_disk=7GB total_vcpus=48 used_vcpus=7 pci_stats=[] stats={'failed_builds': '82', 'num_instances': '7', 'num_vm_building': '7', 'num_task_deleting': '6', 'num_os_type_None': '7', 'num_proj_a5420f4321c84264957a02c02f47649f': '1', 'io_workload': '7', 'num_proj_5683e7f13d7540c1bda6b8e43f1428d0': '2', 'num_proj_846931e260f34a22a2894c7cb88722fa': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1', 'num_proj_ff660b4621c14dc48d0e1fad4c290dcc': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_task_spawning': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1959.144930] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56b7a54e-7cba-4143-b5a4-0948d0655e00 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1959.154108] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58e7e3b3-bc15-44be-b63c-4dcace39bb1f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1959.184094] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36470232-f7cb-44ca-b387-12cfd5866190 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1959.191962] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c73783ca-93bb-42c3-92f3-337ceb87c101 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1959.207017] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1959.215983] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1959.230223] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1959.230437] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.247s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1960.214035] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1960.891978] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1962.893061] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1962.893061] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 1962.893061] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 1962.910278] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1962.910508] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1962.910590] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1962.910682] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1962.910808] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1962.910928] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1962.911064] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 1962.911197] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 1964.892023] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1965.891640] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1965.891885] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 1968.887631] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1970.485086] env[63273]: WARNING oslo_vmware.rw_handles [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1970.485086] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1970.485086] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1970.485086] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1970.485086] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1970.485086] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 1970.485086] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1970.485086] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1970.485086] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1970.485086] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1970.485086] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1970.485086] env[63273]: ERROR oslo_vmware.rw_handles [ 1970.485875] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/6dde67f0-d4f8-401e-a682-18e35874b81b/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1970.487630] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1970.487919] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Copying Virtual Disk [datastore1] vmware_temp/6dde67f0-d4f8-401e-a682-18e35874b81b/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/6dde67f0-d4f8-401e-a682-18e35874b81b/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1970.488263] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-7217ecca-e0cf-4cc2-bd0b-64881ceed4c2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1970.496215] env[63273]: DEBUG oslo_vmware.api [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Waiting for the task: (returnval){ [ 1970.496215] env[63273]: value = "task-5072180" [ 1970.496215] env[63273]: _type = "Task" [ 1970.496215] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1970.505325] env[63273]: DEBUG oslo_vmware.api [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Task: {'id': task-5072180, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1971.006476] env[63273]: DEBUG oslo_vmware.exceptions [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1971.006731] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1971.007319] env[63273]: ERROR nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1971.007319] env[63273]: Faults: ['InvalidArgument'] [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Traceback (most recent call last): [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] yield resources [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] self.driver.spawn(context, instance, image_meta, [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] self._fetch_image_if_missing(context, vi) [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] image_cache(vi, tmp_image_ds_loc) [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] vm_util.copy_virtual_disk( [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] session._wait_for_task(vmdk_copy_task) [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] return self.wait_for_task(task_ref) [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] return evt.wait() [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] result = hub.switch() [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] return self.greenlet.switch() [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] self.f(*self.args, **self.kw) [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] raise exceptions.translate_fault(task_info.error) [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Faults: ['InvalidArgument'] [ 1971.007319] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] [ 1971.008578] env[63273]: INFO nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Terminating instance [ 1971.009314] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1971.009520] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1971.009762] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-bccd33c0-168e-4441-9e8e-17980d0dca25 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.012205] env[63273]: DEBUG nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1971.012405] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1971.013136] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09ffafd7-2abb-491f-8f8d-6fc9282adfb7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.020712] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1971.021017] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-d19762c0-1d2a-454d-bf08-a534791f5216 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.023562] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1971.023743] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1971.024844] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-bf71acac-0ba3-4b8f-81e7-d21ad547e3f4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.030281] env[63273]: DEBUG oslo_vmware.api [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for the task: (returnval){ [ 1971.030281] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52a4a03f-48de-8b66-68f5-8251fc8e17e3" [ 1971.030281] env[63273]: _type = "Task" [ 1971.030281] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1971.039359] env[63273]: DEBUG oslo_vmware.api [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52a4a03f-48de-8b66-68f5-8251fc8e17e3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1971.093782] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1971.095066] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1971.095066] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Deleting the datastore file [datastore1] 61bd6d66-590f-4512-afbf-9abf4a308749 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1971.095066] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-892b10ac-5bea-49c4-aaaa-1a4a771b98fd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.102588] env[63273]: DEBUG oslo_vmware.api [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Waiting for the task: (returnval){ [ 1971.102588] env[63273]: value = "task-5072182" [ 1971.102588] env[63273]: _type = "Task" [ 1971.102588] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1971.112474] env[63273]: DEBUG oslo_vmware.api [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Task: {'id': task-5072182, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1971.541681] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1971.542148] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Creating directory with path [datastore1] vmware_temp/f9bcef8a-2698-4908-a9ab-c67b190963dc/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1971.542148] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-7279dc40-7129-4c69-a6bb-e9a084f4350b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.554187] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Created directory with path [datastore1] vmware_temp/f9bcef8a-2698-4908-a9ab-c67b190963dc/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1971.554374] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Fetch image to [datastore1] vmware_temp/f9bcef8a-2698-4908-a9ab-c67b190963dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1971.554573] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/f9bcef8a-2698-4908-a9ab-c67b190963dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1971.555364] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4acc7f3-45ec-4e54-add3-b21b8807be23 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.562409] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08333f76-8963-4122-acdf-e5320b806591 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.572175] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9fb1070c-c778-4120-b473-1369db117852 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.602936] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0cb9e9f4-20aa-45eb-be11-d30296f2b55e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.614223] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-2446ad41-0c93-4bde-ad48-f653fae4101a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.615922] env[63273]: DEBUG oslo_vmware.api [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Task: {'id': task-5072182, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.064549} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1971.616189] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1971.616372] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1971.616544] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1971.616715] env[63273]: INFO nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1971.618843] env[63273]: DEBUG nova.compute.claims [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1971.619057] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1971.619274] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1971.639458] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1971.706505] env[63273]: DEBUG oslo_vmware.rw_handles [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f9bcef8a-2698-4908-a9ab-c67b190963dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1971.766799] env[63273]: DEBUG oslo_vmware.rw_handles [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1971.767066] env[63273]: DEBUG oslo_vmware.rw_handles [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f9bcef8a-2698-4908-a9ab-c67b190963dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1971.824540] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ede08f8c-6259-49e6-95e8-cf23ae4aa152 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.832255] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36a6dc48-75a3-4027-a180-c48da0b9de6d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.861792] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b851e07-5f4a-491e-bbc1-ef2f1a1f3f06 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.870048] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8db793b8-b99a-438d-acee-21f69bf3b2fb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.885162] env[63273]: DEBUG nova.compute.provider_tree [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1971.893797] env[63273]: DEBUG nova.scheduler.client.report [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1971.907857] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.288s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1971.908412] env[63273]: ERROR nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1971.908412] env[63273]: Faults: ['InvalidArgument'] [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Traceback (most recent call last): [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] self.driver.spawn(context, instance, image_meta, [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] self._fetch_image_if_missing(context, vi) [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] image_cache(vi, tmp_image_ds_loc) [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] vm_util.copy_virtual_disk( [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] session._wait_for_task(vmdk_copy_task) [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] return self.wait_for_task(task_ref) [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] return evt.wait() [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] result = hub.switch() [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] return self.greenlet.switch() [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] self.f(*self.args, **self.kw) [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] raise exceptions.translate_fault(task_info.error) [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Faults: ['InvalidArgument'] [ 1971.908412] env[63273]: ERROR nova.compute.manager [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] [ 1971.909581] env[63273]: DEBUG nova.compute.utils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1971.910693] env[63273]: DEBUG nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Build of instance 61bd6d66-590f-4512-afbf-9abf4a308749 was re-scheduled: A specified parameter was not correct: fileType [ 1971.910693] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 1971.911072] env[63273]: DEBUG nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 1971.911249] env[63273]: DEBUG nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 1971.911420] env[63273]: DEBUG nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1971.911583] env[63273]: DEBUG nova.network.neutron [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1972.268035] env[63273]: DEBUG nova.network.neutron [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1972.283080] env[63273]: INFO nova.compute.manager [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Took 0.37 seconds to deallocate network for instance. [ 1972.382453] env[63273]: INFO nova.scheduler.client.report [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Deleted allocations for instance 61bd6d66-590f-4512-afbf-9abf4a308749 [ 1972.405815] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d06605cc-cb04-4b07-b6bc-dc8ebf33e2af tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "61bd6d66-590f-4512-afbf-9abf4a308749" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 567.968s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1972.406145] env[63273]: DEBUG oslo_concurrency.lockutils [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "61bd6d66-590f-4512-afbf-9abf4a308749" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 371.501s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1972.406370] env[63273]: DEBUG oslo_concurrency.lockutils [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Acquiring lock "61bd6d66-590f-4512-afbf-9abf4a308749-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1972.406578] env[63273]: DEBUG oslo_concurrency.lockutils [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "61bd6d66-590f-4512-afbf-9abf4a308749-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1972.406750] env[63273]: DEBUG oslo_concurrency.lockutils [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "61bd6d66-590f-4512-afbf-9abf4a308749-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1972.408723] env[63273]: INFO nova.compute.manager [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Terminating instance [ 1972.410462] env[63273]: DEBUG nova.compute.manager [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 1972.410638] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1972.411129] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-82042b79-52fd-4f3a-a0fb-c6c524a8d844 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1972.420808] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39442dc3-bc50-4c86-9c69-c806ea19d570 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1972.449549] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 61bd6d66-590f-4512-afbf-9abf4a308749 could not be found. [ 1972.449763] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1972.449944] env[63273]: INFO nova.compute.manager [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1972.450210] env[63273]: DEBUG oslo.service.loopingcall [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1972.450431] env[63273]: DEBUG nova.compute.manager [-] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 1972.450527] env[63273]: DEBUG nova.network.neutron [-] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1972.482557] env[63273]: DEBUG nova.network.neutron [-] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1972.491361] env[63273]: INFO nova.compute.manager [-] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] Took 0.04 seconds to deallocate network for instance. [ 1972.591732] env[63273]: DEBUG oslo_concurrency.lockutils [None req-88de1dbb-35e7-4f8a-9965-1b1a1e3b22f8 tempest-ImagesTestJSON-668207143 tempest-ImagesTestJSON-668207143-project-member] Lock "61bd6d66-590f-4512-afbf-9abf4a308749" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.186s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1972.592801] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "61bd6d66-590f-4512-afbf-9abf4a308749" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 202.355s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1972.593066] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 61bd6d66-590f-4512-afbf-9abf4a308749] During sync_power_state the instance has a pending task (deleting). Skip. [ 1972.593288] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "61bd6d66-590f-4512-afbf-9abf4a308749" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1992.237657] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "fd40e450-e51f-41a9-ba86-7b322f03969f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1992.238023] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fd40e450-e51f-41a9-ba86-7b322f03969f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1992.249282] env[63273]: DEBUG nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 1992.302677] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1992.302944] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1992.304469] env[63273]: INFO nova.compute.claims [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1992.454128] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14b97c78-87be-4c93-8449-0d90735980c8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1992.462436] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c306bd5-1174-4a0d-9bfc-626aa1b84882 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1992.492889] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08fe4b20-7185-4005-8d7f-a3e60e78715c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1992.500477] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1dac413-f4ba-4254-a082-4ddd86d19f57 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1992.514401] env[63273]: DEBUG nova.compute.provider_tree [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1992.522729] env[63273]: DEBUG nova.scheduler.client.report [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1992.537612] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.235s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1992.538177] env[63273]: DEBUG nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 1992.571710] env[63273]: DEBUG nova.compute.utils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1992.573031] env[63273]: DEBUG nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 1992.573210] env[63273]: DEBUG nova.network.neutron [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1992.582498] env[63273]: DEBUG nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 1992.635662] env[63273]: DEBUG nova.policy [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e1ab27df058544c496bd2a078b84e6e8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3387519998ea4ed8a586caeaafed2446', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 1992.648396] env[63273]: DEBUG nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 1992.674657] env[63273]: DEBUG nova.virt.hardware [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1992.675067] env[63273]: DEBUG nova.virt.hardware [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1992.675363] env[63273]: DEBUG nova.virt.hardware [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1992.675692] env[63273]: DEBUG nova.virt.hardware [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1992.675975] env[63273]: DEBUG nova.virt.hardware [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1992.676277] env[63273]: DEBUG nova.virt.hardware [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1992.676622] env[63273]: DEBUG nova.virt.hardware [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1992.676893] env[63273]: DEBUG nova.virt.hardware [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1992.677242] env[63273]: DEBUG nova.virt.hardware [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1992.677558] env[63273]: DEBUG nova.virt.hardware [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1992.677870] env[63273]: DEBUG nova.virt.hardware [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1992.679286] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ae94320-e0b3-4490-98fe-7188f40c752d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1992.691782] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68693612-fb99-4970-9c96-b19371d0c011 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1992.968909] env[63273]: DEBUG nova.network.neutron [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Successfully created port: 3e41f65f-ec60-468a-9824-85393ef642b6 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1993.830400] env[63273]: DEBUG nova.compute.manager [req-5ddea439-fe93-42ae-b5f4-9b9aba7b4a48 req-87caeaad-947c-4be2-9ba5-05aade034519 service nova] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Received event network-vif-plugged-3e41f65f-ec60-468a-9824-85393ef642b6 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1993.830695] env[63273]: DEBUG oslo_concurrency.lockutils [req-5ddea439-fe93-42ae-b5f4-9b9aba7b4a48 req-87caeaad-947c-4be2-9ba5-05aade034519 service nova] Acquiring lock "fd40e450-e51f-41a9-ba86-7b322f03969f-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1993.830840] env[63273]: DEBUG oslo_concurrency.lockutils [req-5ddea439-fe93-42ae-b5f4-9b9aba7b4a48 req-87caeaad-947c-4be2-9ba5-05aade034519 service nova] Lock "fd40e450-e51f-41a9-ba86-7b322f03969f-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1993.831014] env[63273]: DEBUG oslo_concurrency.lockutils [req-5ddea439-fe93-42ae-b5f4-9b9aba7b4a48 req-87caeaad-947c-4be2-9ba5-05aade034519 service nova] Lock "fd40e450-e51f-41a9-ba86-7b322f03969f-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1993.831204] env[63273]: DEBUG nova.compute.manager [req-5ddea439-fe93-42ae-b5f4-9b9aba7b4a48 req-87caeaad-947c-4be2-9ba5-05aade034519 service nova] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] No waiting events found dispatching network-vif-plugged-3e41f65f-ec60-468a-9824-85393ef642b6 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1993.831375] env[63273]: WARNING nova.compute.manager [req-5ddea439-fe93-42ae-b5f4-9b9aba7b4a48 req-87caeaad-947c-4be2-9ba5-05aade034519 service nova] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Received unexpected event network-vif-plugged-3e41f65f-ec60-468a-9824-85393ef642b6 for instance with vm_state building and task_state spawning. [ 1993.917635] env[63273]: DEBUG nova.network.neutron [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Successfully updated port: 3e41f65f-ec60-468a-9824-85393ef642b6 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1993.930448] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "refresh_cache-fd40e450-e51f-41a9-ba86-7b322f03969f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1993.930605] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired lock "refresh_cache-fd40e450-e51f-41a9-ba86-7b322f03969f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1993.930747] env[63273]: DEBUG nova.network.neutron [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1993.972342] env[63273]: DEBUG nova.network.neutron [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1994.148237] env[63273]: DEBUG nova.network.neutron [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Updating instance_info_cache with network_info: [{"id": "3e41f65f-ec60-468a-9824-85393ef642b6", "address": "fa:16:3e:40:b4:ff", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.12", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3e41f65f-ec", "ovs_interfaceid": "3e41f65f-ec60-468a-9824-85393ef642b6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1994.160159] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Releasing lock "refresh_cache-fd40e450-e51f-41a9-ba86-7b322f03969f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1994.160502] env[63273]: DEBUG nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Instance network_info: |[{"id": "3e41f65f-ec60-468a-9824-85393ef642b6", "address": "fa:16:3e:40:b4:ff", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.12", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3e41f65f-ec", "ovs_interfaceid": "3e41f65f-ec60-468a-9824-85393ef642b6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1994.160960] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:40:b4:ff', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'dad4f433-bb0b-45c7-8040-972ef2277f75', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '3e41f65f-ec60-468a-9824-85393ef642b6', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1994.168818] env[63273]: DEBUG oslo.service.loopingcall [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1994.169515] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1994.169795] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-4fc07479-4058-4d20-9feb-90d462a990ac {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1994.191314] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1994.191314] env[63273]: value = "task-5072183" [ 1994.191314] env[63273]: _type = "Task" [ 1994.191314] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1994.200137] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072183, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1994.702271] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072183, 'name': CreateVM_Task} progress is 99%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1995.202439] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072183, 'name': CreateVM_Task, 'duration_secs': 0.605681} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1995.202847] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1995.203359] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1995.203529] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1995.203862] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1995.204130] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-783069c8-3b86-4268-ae74-9fdc75b86227 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1995.208943] env[63273]: DEBUG oslo_vmware.api [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 1995.208943] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5274f497-b1d7-9724-8255-cfc3dfeb21c5" [ 1995.208943] env[63273]: _type = "Task" [ 1995.208943] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1995.217320] env[63273]: DEBUG oslo_vmware.api [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5274f497-b1d7-9724-8255-cfc3dfeb21c5, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1995.721772] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1995.722044] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1995.722272] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1995.861079] env[63273]: DEBUG nova.compute.manager [req-3990d9e7-2f97-4f45-8379-c70329b9cf0c req-dc1f82f4-7f8b-4de7-b65a-812f71d1c81d service nova] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Received event network-changed-3e41f65f-ec60-468a-9824-85393ef642b6 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 1995.861414] env[63273]: DEBUG nova.compute.manager [req-3990d9e7-2f97-4f45-8379-c70329b9cf0c req-dc1f82f4-7f8b-4de7-b65a-812f71d1c81d service nova] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Refreshing instance network info cache due to event network-changed-3e41f65f-ec60-468a-9824-85393ef642b6. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 1995.861666] env[63273]: DEBUG oslo_concurrency.lockutils [req-3990d9e7-2f97-4f45-8379-c70329b9cf0c req-dc1f82f4-7f8b-4de7-b65a-812f71d1c81d service nova] Acquiring lock "refresh_cache-fd40e450-e51f-41a9-ba86-7b322f03969f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1995.861818] env[63273]: DEBUG oslo_concurrency.lockutils [req-3990d9e7-2f97-4f45-8379-c70329b9cf0c req-dc1f82f4-7f8b-4de7-b65a-812f71d1c81d service nova] Acquired lock "refresh_cache-fd40e450-e51f-41a9-ba86-7b322f03969f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1995.861984] env[63273]: DEBUG nova.network.neutron [req-3990d9e7-2f97-4f45-8379-c70329b9cf0c req-dc1f82f4-7f8b-4de7-b65a-812f71d1c81d service nova] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Refreshing network info cache for port 3e41f65f-ec60-468a-9824-85393ef642b6 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1996.134419] env[63273]: DEBUG nova.network.neutron [req-3990d9e7-2f97-4f45-8379-c70329b9cf0c req-dc1f82f4-7f8b-4de7-b65a-812f71d1c81d service nova] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Updated VIF entry in instance network info cache for port 3e41f65f-ec60-468a-9824-85393ef642b6. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1996.134799] env[63273]: DEBUG nova.network.neutron [req-3990d9e7-2f97-4f45-8379-c70329b9cf0c req-dc1f82f4-7f8b-4de7-b65a-812f71d1c81d service nova] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Updating instance_info_cache with network_info: [{"id": "3e41f65f-ec60-468a-9824-85393ef642b6", "address": "fa:16:3e:40:b4:ff", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.12", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3e41f65f-ec", "ovs_interfaceid": "3e41f65f-ec60-468a-9824-85393ef642b6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1996.145636] env[63273]: DEBUG oslo_concurrency.lockutils [req-3990d9e7-2f97-4f45-8379-c70329b9cf0c req-dc1f82f4-7f8b-4de7-b65a-812f71d1c81d service nova] Releasing lock "refresh_cache-fd40e450-e51f-41a9-ba86-7b322f03969f" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2005.635350] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "886b6d46-98a1-4954-9113-0aadfd507d29" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2013.891760] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2017.900310] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2018.891177] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2018.891443] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2018.906654] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2018.906997] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2018.907254] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2018.907540] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2018.908714] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40d6e025-9b22-415e-a3c9-d5ee91a56aba {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2018.917823] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3157460d-188f-4bf3-9595-052222da27ca {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2018.932895] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d55b94c9-544c-4b2d-9bf1-14d6273cef08 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2018.940918] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c200343-b5d3-4950-b2b5-f5c3377b4e52 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2018.971872] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180527MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2018.972037] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2018.972259] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2019.046652] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2019.046819] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 92426c26-52fa-4978-a382-3684ec018bc5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2019.046950] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0afc321d-108c-4c2e-81a7-664091127047 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2019.047090] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2054144d-f068-4d52-b913-617b30d2d1c0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2019.047219] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2019.047343] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 886b6d46-98a1-4954-9113-0aadfd507d29 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2019.047461] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fd40e450-e51f-41a9-ba86-7b322f03969f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2019.047707] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 7 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2019.047865] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1408MB phys_disk=100GB used_disk=7GB total_vcpus=48 used_vcpus=7 pci_stats=[] stats={'failed_builds': '83', 'num_instances': '7', 'num_vm_building': '7', 'num_task_deleting': '6', 'num_os_type_None': '7', 'num_proj_5683e7f13d7540c1bda6b8e43f1428d0': '2', 'io_workload': '7', 'num_proj_846931e260f34a22a2894c7cb88722fa': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '2', 'num_proj_ff660b4621c14dc48d0e1fad4c290dcc': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_task_spawning': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2019.149249] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a575090-85a9-4c73-8a71-6df821ed6a76 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2019.158217] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa7af14a-0bb6-45d2-bd16-b27077a29d39 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2019.187964] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f354afd1-ccca-401b-bf0b-530e7f8e2bc9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2019.196520] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa5c62d3-f3e8-4025-a331-5f53bac80b9d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2019.211695] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2019.221361] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2019.236712] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2019.236913] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.265s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2020.237660] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2020.525149] env[63273]: WARNING oslo_vmware.rw_handles [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2020.525149] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2020.525149] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2020.525149] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2020.525149] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2020.525149] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 2020.525149] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2020.525149] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2020.525149] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2020.525149] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2020.525149] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2020.525149] env[63273]: ERROR oslo_vmware.rw_handles [ 2020.525611] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/f9bcef8a-2698-4908-a9ab-c67b190963dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2020.527867] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2020.528113] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Copying Virtual Disk [datastore1] vmware_temp/f9bcef8a-2698-4908-a9ab-c67b190963dc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/f9bcef8a-2698-4908-a9ab-c67b190963dc/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2020.528398] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-e86a8d94-d189-4dab-89e4-812a74bacdb5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2020.537187] env[63273]: DEBUG oslo_vmware.api [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for the task: (returnval){ [ 2020.537187] env[63273]: value = "task-5072184" [ 2020.537187] env[63273]: _type = "Task" [ 2020.537187] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2020.546080] env[63273]: DEBUG oslo_vmware.api [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Task: {'id': task-5072184, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2020.892187] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2021.047283] env[63273]: DEBUG oslo_vmware.exceptions [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2021.047610] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2021.048183] env[63273]: ERROR nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2021.048183] env[63273]: Faults: ['InvalidArgument'] [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Traceback (most recent call last): [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] yield resources [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] self.driver.spawn(context, instance, image_meta, [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] self._fetch_image_if_missing(context, vi) [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] image_cache(vi, tmp_image_ds_loc) [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] vm_util.copy_virtual_disk( [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] session._wait_for_task(vmdk_copy_task) [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] return self.wait_for_task(task_ref) [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] return evt.wait() [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] result = hub.switch() [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] return self.greenlet.switch() [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] self.f(*self.args, **self.kw) [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] raise exceptions.translate_fault(task_info.error) [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Faults: ['InvalidArgument'] [ 2021.048183] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] [ 2021.048871] env[63273]: INFO nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Terminating instance [ 2021.050576] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2021.050576] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2021.050576] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-dda31fec-3449-40f1-82b2-d5698990aaff {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.052993] env[63273]: DEBUG nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2021.053159] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2021.053916] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16ff61d8-3e48-413f-8967-cca9cb1409b0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.061843] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2021.062139] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0787838d-3921-458f-9f04-13e68f97e86c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.064754] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2021.064933] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2021.066041] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c2d681a1-4327-40de-a26c-0cba275b137a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.071784] env[63273]: DEBUG oslo_vmware.api [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Waiting for the task: (returnval){ [ 2021.071784] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52d7e151-af06-d418-dfff-2229f824ac23" [ 2021.071784] env[63273]: _type = "Task" [ 2021.071784] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2021.084044] env[63273]: DEBUG oslo_vmware.api [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52d7e151-af06-d418-dfff-2229f824ac23, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2021.145115] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2021.145366] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2021.145649] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Deleting the datastore file [datastore1] c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2021.145996] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-537d61df-751a-49f2-b52f-e2b642ace16c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.153808] env[63273]: DEBUG oslo_vmware.api [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for the task: (returnval){ [ 2021.153808] env[63273]: value = "task-5072186" [ 2021.153808] env[63273]: _type = "Task" [ 2021.153808] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2021.162575] env[63273]: DEBUG oslo_vmware.api [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Task: {'id': task-5072186, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2021.583411] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2021.583767] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Creating directory with path [datastore1] vmware_temp/1366d001-4c93-4cc2-a1b6-448f7db499e4/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2021.583952] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-45fe3707-fcff-4ccc-a259-4274a3269db5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.596631] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Created directory with path [datastore1] vmware_temp/1366d001-4c93-4cc2-a1b6-448f7db499e4/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2021.596849] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Fetch image to [datastore1] vmware_temp/1366d001-4c93-4cc2-a1b6-448f7db499e4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2021.597032] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/1366d001-4c93-4cc2-a1b6-448f7db499e4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2021.597840] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-adc58b95-5e10-4bde-be78-77fc2a7cb0d5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.605505] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec699ae2-a724-4232-9d4a-d53175b46f4f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.615286] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d65f906-a9e7-4cb0-8037-0192e46934e8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.647317] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e3fe5bb-4570-43e0-b541-0f7eddcd7185 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.654152] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-be869c00-953b-4a9c-ab8a-3c22298e464e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.664801] env[63273]: DEBUG oslo_vmware.api [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Task: {'id': task-5072186, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.067524} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2021.665073] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2021.665297] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2021.665536] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2021.665729] env[63273]: INFO nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Took 0.61 seconds to destroy the instance on the hypervisor. [ 2021.668111] env[63273]: DEBUG nova.compute.claims [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2021.668302] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2021.668524] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2021.683266] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2021.740871] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1366d001-4c93-4cc2-a1b6-448f7db499e4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2021.799715] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2021.799988] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1366d001-4c93-4cc2-a1b6-448f7db499e4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2021.870382] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02327414-a4f4-46ab-9485-5aee6a7ed19e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.878218] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e5a846fc-7782-466a-bf02-86f72bee1932 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.908166] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35079a5f-4800-44ae-a152-bbea3a6a7e75 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.916165] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5c91d510-825a-4874-9f65-b49c2ecb5d88 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2021.931086] env[63273]: DEBUG nova.compute.provider_tree [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2021.941611] env[63273]: DEBUG nova.scheduler.client.report [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2021.956556] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.288s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2021.957143] env[63273]: ERROR nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2021.957143] env[63273]: Faults: ['InvalidArgument'] [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Traceback (most recent call last): [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] self.driver.spawn(context, instance, image_meta, [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] self._fetch_image_if_missing(context, vi) [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] image_cache(vi, tmp_image_ds_loc) [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] vm_util.copy_virtual_disk( [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] session._wait_for_task(vmdk_copy_task) [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] return self.wait_for_task(task_ref) [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] return evt.wait() [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] result = hub.switch() [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] return self.greenlet.switch() [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] self.f(*self.args, **self.kw) [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] raise exceptions.translate_fault(task_info.error) [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Faults: ['InvalidArgument'] [ 2021.957143] env[63273]: ERROR nova.compute.manager [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] [ 2021.958016] env[63273]: DEBUG nova.compute.utils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2021.959479] env[63273]: DEBUG nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Build of instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc was re-scheduled: A specified parameter was not correct: fileType [ 2021.959479] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 2021.959868] env[63273]: DEBUG nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 2021.960058] env[63273]: DEBUG nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 2021.960236] env[63273]: DEBUG nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2021.960398] env[63273]: DEBUG nova.network.neutron [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2022.560752] env[63273]: DEBUG nova.network.neutron [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2022.574997] env[63273]: INFO nova.compute.manager [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Took 0.61 seconds to deallocate network for instance. [ 2022.677234] env[63273]: INFO nova.scheduler.client.report [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Deleted allocations for instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc [ 2022.716932] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2a93b00f-5022-4b46-be9a-438d9f41a308 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 606.009s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2022.717261] env[63273]: DEBUG oslo_concurrency.lockutils [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 409.891s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2022.717508] env[63273]: DEBUG oslo_concurrency.lockutils [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2022.717722] env[63273]: DEBUG oslo_concurrency.lockutils [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2022.717889] env[63273]: DEBUG oslo_concurrency.lockutils [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2022.720654] env[63273]: INFO nova.compute.manager [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Terminating instance [ 2022.722507] env[63273]: DEBUG nova.compute.manager [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2022.722720] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2022.723036] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ce3bc1bd-daeb-4ed2-99f8-f4aa42737fc3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2022.733759] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e30cb0ef-d507-4bc3-96f0-fbaee74ee247 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2022.764597] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc could not be found. [ 2022.764768] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2022.764948] env[63273]: INFO nova.compute.manager [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2022.765211] env[63273]: DEBUG oslo.service.loopingcall [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2022.765467] env[63273]: DEBUG nova.compute.manager [-] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2022.765564] env[63273]: DEBUG nova.network.neutron [-] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2022.793544] env[63273]: DEBUG nova.network.neutron [-] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2022.803361] env[63273]: INFO nova.compute.manager [-] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] Took 0.04 seconds to deallocate network for instance. [ 2022.892027] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2022.892027] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 2022.892027] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 2022.898021] env[63273]: DEBUG oslo_concurrency.lockutils [None req-18533cc3-9ea2-4f6c-8c3f-b268c82b971e tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.180s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2022.899579] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 252.662s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2022.899773] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc] During sync_power_state the instance has a pending task (deleting). Skip. [ 2022.899952] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "c0db2150-d2d1-4e7e-96bf-e7b49e0aeecc" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2022.910328] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2022.910478] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2022.910610] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2022.910761] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2022.910862] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2022.910986] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2022.911128] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 2024.892362] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2024.892716] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances with incomplete migration {{(pid=63273) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11345}} [ 2025.902561] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2026.891722] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2026.891875] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 2029.136686] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "415fad0a-0af4-45b0-a958-ff900fbcb05b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2029.137170] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "415fad0a-0af4-45b0-a958-ff900fbcb05b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2029.148409] env[63273]: DEBUG nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 2029.202168] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2029.202441] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2029.204078] env[63273]: INFO nova.compute.claims [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2029.353251] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba0d0e1f-b6f2-4296-bb5b-424800d02646 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2029.361919] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d322d681-82d0-412a-95a6-09bed541c064 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2029.393912] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0c04a75-7079-422c-8ca4-690010769613 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2029.402605] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-455186fa-7203-41ac-986e-7fe760b6f721 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2029.417042] env[63273]: DEBUG nova.compute.provider_tree [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2029.426135] env[63273]: DEBUG nova.scheduler.client.report [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2029.441709] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.239s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2029.442133] env[63273]: DEBUG nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 2029.480793] env[63273]: DEBUG nova.compute.utils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2029.483059] env[63273]: DEBUG nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 2029.483895] env[63273]: DEBUG nova.network.neutron [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 2029.495656] env[63273]: DEBUG nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 2029.564742] env[63273]: DEBUG nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 2029.571122] env[63273]: DEBUG nova.policy [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '611c75633a8443e389b4736d21f6ca58', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ff660b4621c14dc48d0e1fad4c290dcc', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 2029.591568] env[63273]: DEBUG nova.virt.hardware [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 2029.592292] env[63273]: DEBUG nova.virt.hardware [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 2029.592292] env[63273]: DEBUG nova.virt.hardware [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2029.592421] env[63273]: DEBUG nova.virt.hardware [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 2029.592527] env[63273]: DEBUG nova.virt.hardware [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2029.592668] env[63273]: DEBUG nova.virt.hardware [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 2029.592930] env[63273]: DEBUG nova.virt.hardware [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 2029.593132] env[63273]: DEBUG nova.virt.hardware [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 2029.593310] env[63273]: DEBUG nova.virt.hardware [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 2029.593478] env[63273]: DEBUG nova.virt.hardware [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 2029.593650] env[63273]: DEBUG nova.virt.hardware [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 2029.594672] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4261449-4bf4-4dde-8f48-f6605138d9ab {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2029.603155] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8d0b1c6-5186-4a02-b23a-ecaf752af49a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2029.891550] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2029.891837] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11307}} [ 2029.901872] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] There are 0 instances to clean {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11316}} [ 2029.952645] env[63273]: DEBUG nova.network.neutron [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Successfully created port: c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2030.639675] env[63273]: DEBUG nova.compute.manager [req-aeea3f1d-3942-4243-8a7f-226a8c917f68 req-d7a3a3fc-ebbe-43e3-9bf6-97a08679d3a1 service nova] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Received event network-vif-plugged-c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 2030.639971] env[63273]: DEBUG oslo_concurrency.lockutils [req-aeea3f1d-3942-4243-8a7f-226a8c917f68 req-d7a3a3fc-ebbe-43e3-9bf6-97a08679d3a1 service nova] Acquiring lock "415fad0a-0af4-45b0-a958-ff900fbcb05b-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2030.640211] env[63273]: DEBUG oslo_concurrency.lockutils [req-aeea3f1d-3942-4243-8a7f-226a8c917f68 req-d7a3a3fc-ebbe-43e3-9bf6-97a08679d3a1 service nova] Lock "415fad0a-0af4-45b0-a958-ff900fbcb05b-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2030.640386] env[63273]: DEBUG oslo_concurrency.lockutils [req-aeea3f1d-3942-4243-8a7f-226a8c917f68 req-d7a3a3fc-ebbe-43e3-9bf6-97a08679d3a1 service nova] Lock "415fad0a-0af4-45b0-a958-ff900fbcb05b-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2030.640561] env[63273]: DEBUG nova.compute.manager [req-aeea3f1d-3942-4243-8a7f-226a8c917f68 req-d7a3a3fc-ebbe-43e3-9bf6-97a08679d3a1 service nova] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] No waiting events found dispatching network-vif-plugged-c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2030.640727] env[63273]: WARNING nova.compute.manager [req-aeea3f1d-3942-4243-8a7f-226a8c917f68 req-d7a3a3fc-ebbe-43e3-9bf6-97a08679d3a1 service nova] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Received unexpected event network-vif-plugged-c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0 for instance with vm_state building and task_state spawning. [ 2030.737489] env[63273]: DEBUG nova.network.neutron [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Successfully updated port: c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2030.752024] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "refresh_cache-415fad0a-0af4-45b0-a958-ff900fbcb05b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2030.752222] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquired lock "refresh_cache-415fad0a-0af4-45b0-a958-ff900fbcb05b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2030.752329] env[63273]: DEBUG nova.network.neutron [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2030.800149] env[63273]: DEBUG nova.network.neutron [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2030.897204] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2030.990525] env[63273]: DEBUG nova.network.neutron [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Updating instance_info_cache with network_info: [{"id": "c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0", "address": "fa:16:3e:c6:af:65", "network": {"id": "f04a4851-a302-4dcf-97cb-b3c86b22cd29", "bridge": "br-int", "label": "tempest-SecurityGroupsTestJSON-1730702358-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff660b4621c14dc48d0e1fad4c290dcc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "62f28d75-4e6a-4ae5-b8b3-d0652ea26d08", "external-id": "nsx-vlan-transportzone-801", "segmentation_id": 801, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc8a4f7b9-68", "ovs_interfaceid": "c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2031.001858] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Releasing lock "refresh_cache-415fad0a-0af4-45b0-a958-ff900fbcb05b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2031.002181] env[63273]: DEBUG nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Instance network_info: |[{"id": "c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0", "address": "fa:16:3e:c6:af:65", "network": {"id": "f04a4851-a302-4dcf-97cb-b3c86b22cd29", "bridge": "br-int", "label": "tempest-SecurityGroupsTestJSON-1730702358-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff660b4621c14dc48d0e1fad4c290dcc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "62f28d75-4e6a-4ae5-b8b3-d0652ea26d08", "external-id": "nsx-vlan-transportzone-801", "segmentation_id": 801, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc8a4f7b9-68", "ovs_interfaceid": "c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2031.002603] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:c6:af:65', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '62f28d75-4e6a-4ae5-b8b3-d0652ea26d08', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2031.010211] env[63273]: DEBUG oslo.service.loopingcall [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2031.010739] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2031.010980] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-b64996ce-c579-4922-a463-68eca8103ac1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2031.032441] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2031.032441] env[63273]: value = "task-5072187" [ 2031.032441] env[63273]: _type = "Task" [ 2031.032441] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2031.041271] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072187, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2031.544663] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072187, 'name': CreateVM_Task, 'duration_secs': 0.300044} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2031.544799] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2031.545579] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2031.545752] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2031.546096] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2031.546423] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-6b2379f8-79ea-4365-9beb-18f288c18fd3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2031.551545] env[63273]: DEBUG oslo_vmware.api [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for the task: (returnval){ [ 2031.551545] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52705cae-7349-63ad-3399-31e9662b5aba" [ 2031.551545] env[63273]: _type = "Task" [ 2031.551545] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2031.560402] env[63273]: DEBUG oslo_vmware.api [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52705cae-7349-63ad-3399-31e9662b5aba, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2032.065291] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2032.065745] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2032.065983] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2032.676262] env[63273]: DEBUG nova.compute.manager [req-e3f40d26-2086-4e31-9ce0-04c52614a701 req-ac8b884f-924d-4438-8206-6a7e52c4e15f service nova] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Received event network-changed-c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 2032.676262] env[63273]: DEBUG nova.compute.manager [req-e3f40d26-2086-4e31-9ce0-04c52614a701 req-ac8b884f-924d-4438-8206-6a7e52c4e15f service nova] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Refreshing instance network info cache due to event network-changed-c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 2032.676262] env[63273]: DEBUG oslo_concurrency.lockutils [req-e3f40d26-2086-4e31-9ce0-04c52614a701 req-ac8b884f-924d-4438-8206-6a7e52c4e15f service nova] Acquiring lock "refresh_cache-415fad0a-0af4-45b0-a958-ff900fbcb05b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2032.676262] env[63273]: DEBUG oslo_concurrency.lockutils [req-e3f40d26-2086-4e31-9ce0-04c52614a701 req-ac8b884f-924d-4438-8206-6a7e52c4e15f service nova] Acquired lock "refresh_cache-415fad0a-0af4-45b0-a958-ff900fbcb05b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2032.676534] env[63273]: DEBUG nova.network.neutron [req-e3f40d26-2086-4e31-9ce0-04c52614a701 req-ac8b884f-924d-4438-8206-6a7e52c4e15f service nova] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Refreshing network info cache for port c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2032.948837] env[63273]: DEBUG nova.network.neutron [req-e3f40d26-2086-4e31-9ce0-04c52614a701 req-ac8b884f-924d-4438-8206-6a7e52c4e15f service nova] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Updated VIF entry in instance network info cache for port c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 2032.949228] env[63273]: DEBUG nova.network.neutron [req-e3f40d26-2086-4e31-9ce0-04c52614a701 req-ac8b884f-924d-4438-8206-6a7e52c4e15f service nova] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Updating instance_info_cache with network_info: [{"id": "c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0", "address": "fa:16:3e:c6:af:65", "network": {"id": "f04a4851-a302-4dcf-97cb-b3c86b22cd29", "bridge": "br-int", "label": "tempest-SecurityGroupsTestJSON-1730702358-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff660b4621c14dc48d0e1fad4c290dcc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "62f28d75-4e6a-4ae5-b8b3-d0652ea26d08", "external-id": "nsx-vlan-transportzone-801", "segmentation_id": 801, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc8a4f7b9-68", "ovs_interfaceid": "c8a4f7b9-6890-4f73-b1ab-4b53f8a444b0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2032.959087] env[63273]: DEBUG oslo_concurrency.lockutils [req-e3f40d26-2086-4e31-9ce0-04c52614a701 req-ac8b884f-924d-4438-8206-6a7e52c4e15f service nova] Releasing lock "refresh_cache-415fad0a-0af4-45b0-a958-ff900fbcb05b" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2068.745987] env[63273]: WARNING oslo_vmware.rw_handles [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2068.745987] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2068.745987] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2068.745987] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2068.745987] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2068.745987] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 2068.745987] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2068.745987] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2068.745987] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2068.745987] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2068.745987] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2068.745987] env[63273]: ERROR oslo_vmware.rw_handles [ 2068.746698] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/1366d001-4c93-4cc2-a1b6-448f7db499e4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2068.748132] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2068.748366] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Copying Virtual Disk [datastore1] vmware_temp/1366d001-4c93-4cc2-a1b6-448f7db499e4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/1366d001-4c93-4cc2-a1b6-448f7db499e4/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2068.748658] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-8585a555-a346-453f-a2b7-7961705a32a6 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.756783] env[63273]: DEBUG oslo_vmware.api [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Waiting for the task: (returnval){ [ 2068.756783] env[63273]: value = "task-5072188" [ 2068.756783] env[63273]: _type = "Task" [ 2068.756783] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2068.765323] env[63273]: DEBUG oslo_vmware.api [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Task: {'id': task-5072188, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2069.268056] env[63273]: DEBUG oslo_vmware.exceptions [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2069.268056] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2069.268467] env[63273]: ERROR nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2069.268467] env[63273]: Faults: ['InvalidArgument'] [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Traceback (most recent call last): [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] yield resources [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] self.driver.spawn(context, instance, image_meta, [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] self._fetch_image_if_missing(context, vi) [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] image_cache(vi, tmp_image_ds_loc) [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] vm_util.copy_virtual_disk( [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] session._wait_for_task(vmdk_copy_task) [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] return self.wait_for_task(task_ref) [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] return evt.wait() [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] result = hub.switch() [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] return self.greenlet.switch() [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] self.f(*self.args, **self.kw) [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] raise exceptions.translate_fault(task_info.error) [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Faults: ['InvalidArgument'] [ 2069.268467] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] [ 2069.269296] env[63273]: INFO nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Terminating instance [ 2069.270418] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2069.270624] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2069.270874] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-7709b3c5-66a7-4b55-940f-b18ecf47c5b2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.273435] env[63273]: DEBUG nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2069.273644] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2069.274430] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d72a2cc-a02c-46de-9d9f-5d83b86df849 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.282138] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2069.282474] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-e676c97a-9ab8-4941-b1c0-ccafa9ca4c05 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.284940] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2069.285191] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2069.286246] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-93fc522c-32b1-4f6b-8ed4-40eff34926e4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.291593] env[63273]: DEBUG oslo_vmware.api [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 2069.291593] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5203f43d-23a8-7586-4348-d46feb8ab620" [ 2069.291593] env[63273]: _type = "Task" [ 2069.291593] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2069.299365] env[63273]: DEBUG oslo_vmware.api [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5203f43d-23a8-7586-4348-d46feb8ab620, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2069.358048] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2069.358311] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2069.358428] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Deleting the datastore file [datastore1] 92426c26-52fa-4978-a382-3684ec018bc5 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2069.358678] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-6068a312-48c4-4f81-8180-b8a4654fdfea {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.365541] env[63273]: DEBUG oslo_vmware.api [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Waiting for the task: (returnval){ [ 2069.365541] env[63273]: value = "task-5072190" [ 2069.365541] env[63273]: _type = "Task" [ 2069.365541] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2069.374772] env[63273]: DEBUG oslo_vmware.api [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Task: {'id': task-5072190, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2069.802251] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2069.802640] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Creating directory with path [datastore1] vmware_temp/f14885c7-d537-4e34-96f3-ea0754e26fce/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2069.802777] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-db387838-4405-4d4a-af3b-5c082812375f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.815591] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Created directory with path [datastore1] vmware_temp/f14885c7-d537-4e34-96f3-ea0754e26fce/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2069.815868] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Fetch image to [datastore1] vmware_temp/f14885c7-d537-4e34-96f3-ea0754e26fce/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2069.816063] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/f14885c7-d537-4e34-96f3-ea0754e26fce/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2069.816849] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ac58415-ae56-441d-bf89-5a162265dfda {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.824639] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e729e5f-b782-40e5-8be0-80e70a0bc7d4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.835674] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9763b44-c466-411c-b1d3-c176f71a7c10 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.870013] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bfce8472-612e-4895-ac68-c5289e1233cc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.877533] env[63273]: DEBUG oslo_vmware.api [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Task: {'id': task-5072190, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.065599} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2069.879036] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2069.879238] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2069.879413] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2069.879586] env[63273]: INFO nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Took 0.61 seconds to destroy the instance on the hypervisor. [ 2069.881418] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-be8458f5-aa74-4c48-813f-c46e702bc632 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.883372] env[63273]: DEBUG nova.compute.claims [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2069.883535] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2069.883755] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2069.911792] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2069.973469] env[63273]: DEBUG oslo_vmware.rw_handles [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f14885c7-d537-4e34-96f3-ea0754e26fce/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2070.036170] env[63273]: DEBUG oslo_vmware.rw_handles [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2070.036384] env[63273]: DEBUG oslo_vmware.rw_handles [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f14885c7-d537-4e34-96f3-ea0754e26fce/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2070.097982] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69fb1b88-feed-479d-ab62-df3d602b858c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2070.106823] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e9572b3c-6907-4b05-abdb-127a470ee78e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2070.138988] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18a515af-9074-4152-8c7c-8d9ac156ea3c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2070.147235] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6b9b79d-6e9f-4455-bf5a-77ff6b8ea32d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2070.161178] env[63273]: DEBUG nova.compute.provider_tree [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2070.171527] env[63273]: DEBUG nova.scheduler.client.report [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2070.184791] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.301s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2070.185339] env[63273]: ERROR nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2070.185339] env[63273]: Faults: ['InvalidArgument'] [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Traceback (most recent call last): [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] self.driver.spawn(context, instance, image_meta, [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] self._fetch_image_if_missing(context, vi) [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] image_cache(vi, tmp_image_ds_loc) [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] vm_util.copy_virtual_disk( [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] session._wait_for_task(vmdk_copy_task) [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] return self.wait_for_task(task_ref) [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] return evt.wait() [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] result = hub.switch() [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] return self.greenlet.switch() [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] self.f(*self.args, **self.kw) [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] raise exceptions.translate_fault(task_info.error) [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Faults: ['InvalidArgument'] [ 2070.185339] env[63273]: ERROR nova.compute.manager [instance: 92426c26-52fa-4978-a382-3684ec018bc5] [ 2070.186173] env[63273]: DEBUG nova.compute.utils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2070.187544] env[63273]: DEBUG nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Build of instance 92426c26-52fa-4978-a382-3684ec018bc5 was re-scheduled: A specified parameter was not correct: fileType [ 2070.187544] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 2070.187914] env[63273]: DEBUG nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 2070.188105] env[63273]: DEBUG nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 2070.188277] env[63273]: DEBUG nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2070.188436] env[63273]: DEBUG nova.network.neutron [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2070.572475] env[63273]: DEBUG nova.network.neutron [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2070.586856] env[63273]: INFO nova.compute.manager [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Took 0.40 seconds to deallocate network for instance. [ 2070.677485] env[63273]: INFO nova.scheduler.client.report [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Deleted allocations for instance 92426c26-52fa-4978-a382-3684ec018bc5 [ 2070.700014] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f74f9361-686d-4e0b-9c95-aed6035a50bd tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Lock "92426c26-52fa-4978-a382-3684ec018bc5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 517.429s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2070.700319] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Lock "92426c26-52fa-4978-a382-3684ec018bc5" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 321.999s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2070.700533] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Acquiring lock "92426c26-52fa-4978-a382-3684ec018bc5-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2070.700738] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Lock "92426c26-52fa-4978-a382-3684ec018bc5-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2070.700915] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Lock "92426c26-52fa-4978-a382-3684ec018bc5-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2070.703131] env[63273]: INFO nova.compute.manager [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Terminating instance [ 2070.705724] env[63273]: DEBUG nova.compute.manager [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2070.705977] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2070.706283] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-cd8dc0da-c45b-4f79-b081-23a78d192329 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2070.719701] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-390b7de6-1243-407b-b178-64963c81617c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2070.750961] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 92426c26-52fa-4978-a382-3684ec018bc5 could not be found. [ 2070.751207] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2070.751389] env[63273]: INFO nova.compute.manager [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Took 0.05 seconds to destroy the instance on the hypervisor. [ 2070.751636] env[63273]: DEBUG oslo.service.loopingcall [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2070.751860] env[63273]: DEBUG nova.compute.manager [-] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2070.751964] env[63273]: DEBUG nova.network.neutron [-] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2070.777627] env[63273]: DEBUG nova.network.neutron [-] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2070.785793] env[63273]: INFO nova.compute.manager [-] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] Took 0.03 seconds to deallocate network for instance. [ 2070.878617] env[63273]: DEBUG oslo_concurrency.lockutils [None req-bb2ae772-1cec-4747-8db1-a9f5eed22dc7 tempest-ServerRescueTestJSONUnderV235-720662927 tempest-ServerRescueTestJSONUnderV235-720662927-project-member] Lock "92426c26-52fa-4978-a382-3684ec018bc5" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.178s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2070.879620] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "92426c26-52fa-4978-a382-3684ec018bc5" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 300.642s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2070.879814] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 92426c26-52fa-4978-a382-3684ec018bc5] During sync_power_state the instance has a pending task (deleting). Skip. [ 2070.879990] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "92426c26-52fa-4978-a382-3684ec018bc5" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2076.049547] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "8afeca90-bf18-45ad-8b7e-d1a22632e0b0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2076.049547] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "8afeca90-bf18-45ad-8b7e-d1a22632e0b0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2076.060643] env[63273]: DEBUG nova.compute.manager [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 2076.155550] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2076.155818] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2076.157421] env[63273]: INFO nova.compute.claims [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2076.223726] env[63273]: DEBUG nova.scheduler.client.report [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Refreshing inventories for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 2076.239782] env[63273]: DEBUG nova.scheduler.client.report [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Updating ProviderTree inventory for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 2076.240307] env[63273]: DEBUG nova.compute.provider_tree [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Updating inventory in ProviderTree for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2076.251887] env[63273]: DEBUG nova.scheduler.client.report [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Refreshing aggregate associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, aggregates: None {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 2076.271461] env[63273]: DEBUG nova.scheduler.client.report [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Refreshing trait associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NODE,COMPUTE_IMAGE_TYPE_ISO {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 2076.357764] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-186e7350-9e7c-434f-881a-dedf09f2c733 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2076.365730] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e7c71e3-6e7b-49df-bc2f-f52b31cd58fc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2076.394541] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46968a11-737f-4c41-85d0-de87654eced9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2076.401978] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8f8b72b-3abb-4119-9f17-988680b77d43 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2076.416256] env[63273]: DEBUG nova.compute.provider_tree [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2076.424857] env[63273]: DEBUG nova.scheduler.client.report [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2076.439370] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.283s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2076.439872] env[63273]: DEBUG nova.compute.manager [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 2076.471367] env[63273]: DEBUG nova.compute.utils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2076.473193] env[63273]: DEBUG nova.compute.manager [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 2076.473463] env[63273]: DEBUG nova.network.neutron [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 2076.482743] env[63273]: DEBUG nova.compute.manager [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 2076.538918] env[63273]: DEBUG nova.policy [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7648b8a75224424f8226b574da373a87', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '495ae26325184812a744fc95da67038b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 2076.547810] env[63273]: DEBUG nova.compute.manager [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 2076.573540] env[63273]: DEBUG nova.virt.hardware [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 2076.573781] env[63273]: DEBUG nova.virt.hardware [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 2076.573938] env[63273]: DEBUG nova.virt.hardware [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2076.574135] env[63273]: DEBUG nova.virt.hardware [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 2076.574285] env[63273]: DEBUG nova.virt.hardware [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2076.574438] env[63273]: DEBUG nova.virt.hardware [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 2076.574686] env[63273]: DEBUG nova.virt.hardware [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 2076.574852] env[63273]: DEBUG nova.virt.hardware [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 2076.575033] env[63273]: DEBUG nova.virt.hardware [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 2076.575207] env[63273]: DEBUG nova.virt.hardware [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 2076.575414] env[63273]: DEBUG nova.virt.hardware [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 2076.576289] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa46a6f6-2298-450c-9a9d-02cf54331497 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2076.584881] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a683d487-f2b5-4753-bc10-6e457cf9f2b5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2077.085239] env[63273]: DEBUG nova.network.neutron [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Successfully created port: 09fa46e6-a8c5-401f-bbf6-18dddaba9995 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2077.644404] env[63273]: DEBUG nova.compute.manager [req-202de1e9-ee03-400b-8086-a7cbf788bb69 req-0a889977-5f7d-4404-806c-2480cbf89edf service nova] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Received event network-vif-plugged-09fa46e6-a8c5-401f-bbf6-18dddaba9995 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 2077.644638] env[63273]: DEBUG oslo_concurrency.lockutils [req-202de1e9-ee03-400b-8086-a7cbf788bb69 req-0a889977-5f7d-4404-806c-2480cbf89edf service nova] Acquiring lock "8afeca90-bf18-45ad-8b7e-d1a22632e0b0-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2077.644854] env[63273]: DEBUG oslo_concurrency.lockutils [req-202de1e9-ee03-400b-8086-a7cbf788bb69 req-0a889977-5f7d-4404-806c-2480cbf89edf service nova] Lock "8afeca90-bf18-45ad-8b7e-d1a22632e0b0-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2077.645128] env[63273]: DEBUG oslo_concurrency.lockutils [req-202de1e9-ee03-400b-8086-a7cbf788bb69 req-0a889977-5f7d-4404-806c-2480cbf89edf service nova] Lock "8afeca90-bf18-45ad-8b7e-d1a22632e0b0-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2077.645385] env[63273]: DEBUG nova.compute.manager [req-202de1e9-ee03-400b-8086-a7cbf788bb69 req-0a889977-5f7d-4404-806c-2480cbf89edf service nova] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] No waiting events found dispatching network-vif-plugged-09fa46e6-a8c5-401f-bbf6-18dddaba9995 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2077.645624] env[63273]: WARNING nova.compute.manager [req-202de1e9-ee03-400b-8086-a7cbf788bb69 req-0a889977-5f7d-4404-806c-2480cbf89edf service nova] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Received unexpected event network-vif-plugged-09fa46e6-a8c5-401f-bbf6-18dddaba9995 for instance with vm_state building and task_state spawning. [ 2077.730201] env[63273]: DEBUG nova.network.neutron [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Successfully updated port: 09fa46e6-a8c5-401f-bbf6-18dddaba9995 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2077.747670] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "refresh_cache-8afeca90-bf18-45ad-8b7e-d1a22632e0b0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2077.748017] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "refresh_cache-8afeca90-bf18-45ad-8b7e-d1a22632e0b0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2077.748017] env[63273]: DEBUG nova.network.neutron [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2077.797647] env[63273]: DEBUG nova.network.neutron [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2077.984290] env[63273]: DEBUG nova.network.neutron [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Updating instance_info_cache with network_info: [{"id": "09fa46e6-a8c5-401f-bbf6-18dddaba9995", "address": "fa:16:3e:b4:86:5a", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap09fa46e6-a8", "ovs_interfaceid": "09fa46e6-a8c5-401f-bbf6-18dddaba9995", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2077.996748] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "refresh_cache-8afeca90-bf18-45ad-8b7e-d1a22632e0b0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2077.997190] env[63273]: DEBUG nova.compute.manager [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Instance network_info: |[{"id": "09fa46e6-a8c5-401f-bbf6-18dddaba9995", "address": "fa:16:3e:b4:86:5a", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap09fa46e6-a8", "ovs_interfaceid": "09fa46e6-a8c5-401f-bbf6-18dddaba9995", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2077.997554] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:b4:86:5a', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '69e1a5c1-4ae7-409b-8de7-d401684ef60d', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '09fa46e6-a8c5-401f-bbf6-18dddaba9995', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2078.005303] env[63273]: DEBUG oslo.service.loopingcall [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2078.006296] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2078.006526] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3476daf0-cf45-44dd-aca3-0500f372c190 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2078.027352] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2078.027352] env[63273]: value = "task-5072191" [ 2078.027352] env[63273]: _type = "Task" [ 2078.027352] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2078.035832] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072191, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2078.537693] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072191, 'name': CreateVM_Task, 'duration_secs': 0.288389} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2078.538070] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2078.538593] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2078.538758] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2078.539146] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2078.539408] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d9def7ea-cac8-4f35-b5cb-2d9c48bbf3ec {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2078.544143] env[63273]: DEBUG oslo_vmware.api [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 2078.544143] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]525e4bc0-1015-e549-1997-ba3c088db856" [ 2078.544143] env[63273]: _type = "Task" [ 2078.544143] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2078.551933] env[63273]: DEBUG oslo_vmware.api [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]525e4bc0-1015-e549-1997-ba3c088db856, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2079.055916] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2079.057028] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2079.057028] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2079.676936] env[63273]: DEBUG nova.compute.manager [req-2deb5b74-2412-4286-917e-35c19006742b req-8b87c5cc-5f10-4c45-bc3e-b5e69007012b service nova] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Received event network-changed-09fa46e6-a8c5-401f-bbf6-18dddaba9995 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 2079.677195] env[63273]: DEBUG nova.compute.manager [req-2deb5b74-2412-4286-917e-35c19006742b req-8b87c5cc-5f10-4c45-bc3e-b5e69007012b service nova] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Refreshing instance network info cache due to event network-changed-09fa46e6-a8c5-401f-bbf6-18dddaba9995. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 2079.677397] env[63273]: DEBUG oslo_concurrency.lockutils [req-2deb5b74-2412-4286-917e-35c19006742b req-8b87c5cc-5f10-4c45-bc3e-b5e69007012b service nova] Acquiring lock "refresh_cache-8afeca90-bf18-45ad-8b7e-d1a22632e0b0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2079.677597] env[63273]: DEBUG oslo_concurrency.lockutils [req-2deb5b74-2412-4286-917e-35c19006742b req-8b87c5cc-5f10-4c45-bc3e-b5e69007012b service nova] Acquired lock "refresh_cache-8afeca90-bf18-45ad-8b7e-d1a22632e0b0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2079.677875] env[63273]: DEBUG nova.network.neutron [req-2deb5b74-2412-4286-917e-35c19006742b req-8b87c5cc-5f10-4c45-bc3e-b5e69007012b service nova] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Refreshing network info cache for port 09fa46e6-a8c5-401f-bbf6-18dddaba9995 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2079.891008] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2079.932952] env[63273]: DEBUG nova.network.neutron [req-2deb5b74-2412-4286-917e-35c19006742b req-8b87c5cc-5f10-4c45-bc3e-b5e69007012b service nova] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Updated VIF entry in instance network info cache for port 09fa46e6-a8c5-401f-bbf6-18dddaba9995. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 2079.933331] env[63273]: DEBUG nova.network.neutron [req-2deb5b74-2412-4286-917e-35c19006742b req-8b87c5cc-5f10-4c45-bc3e-b5e69007012b service nova] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Updating instance_info_cache with network_info: [{"id": "09fa46e6-a8c5-401f-bbf6-18dddaba9995", "address": "fa:16:3e:b4:86:5a", "network": {"id": "ad015bd1-2b91-454e-a9e5-7e61253eb8bf", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-570249617-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "495ae26325184812a744fc95da67038b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "69e1a5c1-4ae7-409b-8de7-d401684ef60d", "external-id": "nsx-vlan-transportzone-740", "segmentation_id": 740, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap09fa46e6-a8", "ovs_interfaceid": "09fa46e6-a8c5-401f-bbf6-18dddaba9995", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2079.943050] env[63273]: DEBUG oslo_concurrency.lockutils [req-2deb5b74-2412-4286-917e-35c19006742b req-8b87c5cc-5f10-4c45-bc3e-b5e69007012b service nova] Releasing lock "refresh_cache-8afeca90-bf18-45ad-8b7e-d1a22632e0b0" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2080.886760] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2080.906329] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2080.906528] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2080.920360] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2080.920579] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2080.920745] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2080.920898] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2080.921999] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d68bf91-1d9a-476d-852e-7790f006b6a1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2080.930944] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7e1626b-9253-4bf8-8dd8-ff83affcaa54 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2080.944910] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-187c0d13-09c5-4eaa-917e-b1be552a8bbf {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2080.951548] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1cffaa72-90d2-4ece-ab75-97bfc8a90b4e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2080.979824] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180527MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2080.979974] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2080.980195] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2081.066054] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 0afc321d-108c-4c2e-81a7-664091127047 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2081.066229] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2054144d-f068-4d52-b913-617b30d2d1c0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2081.066376] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2081.066506] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 886b6d46-98a1-4954-9113-0aadfd507d29 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2081.066629] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fd40e450-e51f-41a9-ba86-7b322f03969f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2081.066752] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 415fad0a-0af4-45b0-a958-ff900fbcb05b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2081.066871] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8afeca90-bf18-45ad-8b7e-d1a22632e0b0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2081.067136] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 7 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2081.067319] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1408MB phys_disk=100GB used_disk=7GB total_vcpus=48 used_vcpus=7 pci_stats=[] stats={'failed_builds': '85', 'num_instances': '7', 'num_vm_building': '7', 'num_task_deleting': '4', 'num_os_type_None': '7', 'num_proj_3387519998ea4ed8a586caeaafed2446': '2', 'io_workload': '7', 'num_proj_ff660b4621c14dc48d0e1fad4c290dcc': '2', 'num_proj_495ae26325184812a744fc95da67038b': '2', 'num_proj_5683e7f13d7540c1bda6b8e43f1428d0': '1', 'num_task_spawning': '3'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2081.156343] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b8913fd-f564-463c-8bd5-7a7fb28ea2df {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2081.164082] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc23c23b-0c2b-442e-8379-0877be8e7a06 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2081.193353] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1555fa8a-c391-406f-bd2d-9f20393ce614 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2081.201029] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-859568b3-0814-4e22-8021-b7c674da22a3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2081.214144] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2081.223262] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2081.242911] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2081.243176] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.263s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2082.228671] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2082.891354] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2083.892094] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2083.892502] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 2083.892502] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 2083.910134] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2083.910134] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2083.910134] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2083.910348] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2083.910469] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2083.910523] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2083.910629] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2083.910752] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 2086.892198] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2086.892537] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2086.892580] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 2092.887212] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2115.558022] env[63273]: WARNING oslo_vmware.rw_handles [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2115.558022] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2115.558022] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2115.558022] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2115.558022] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2115.558022] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 2115.558022] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2115.558022] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2115.558022] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2115.558022] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2115.558022] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2115.558022] env[63273]: ERROR oslo_vmware.rw_handles [ 2115.558697] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/f14885c7-d537-4e34-96f3-ea0754e26fce/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2115.560499] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2115.560784] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Copying Virtual Disk [datastore1] vmware_temp/f14885c7-d537-4e34-96f3-ea0754e26fce/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/f14885c7-d537-4e34-96f3-ea0754e26fce/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2115.561110] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-b2e2fe09-824f-4252-a33c-e32806cf1670 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2115.570050] env[63273]: DEBUG oslo_vmware.api [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 2115.570050] env[63273]: value = "task-5072192" [ 2115.570050] env[63273]: _type = "Task" [ 2115.570050] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2115.578640] env[63273]: DEBUG oslo_vmware.api [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': task-5072192, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2116.081611] env[63273]: DEBUG oslo_vmware.exceptions [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2116.081914] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2116.082482] env[63273]: ERROR nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2116.082482] env[63273]: Faults: ['InvalidArgument'] [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] Traceback (most recent call last): [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] yield resources [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] self.driver.spawn(context, instance, image_meta, [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] self._fetch_image_if_missing(context, vi) [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] image_cache(vi, tmp_image_ds_loc) [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] vm_util.copy_virtual_disk( [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] session._wait_for_task(vmdk_copy_task) [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] return self.wait_for_task(task_ref) [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] return evt.wait() [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] result = hub.switch() [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] return self.greenlet.switch() [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] self.f(*self.args, **self.kw) [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] raise exceptions.translate_fault(task_info.error) [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] Faults: ['InvalidArgument'] [ 2116.082482] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] [ 2116.083571] env[63273]: INFO nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Terminating instance [ 2116.085220] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2116.085220] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2116.085220] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-1bc600f9-4f1f-4d22-9bc6-6be45479ad24 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.087246] env[63273]: DEBUG nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2116.087434] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2116.088179] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b2c16a7-0213-4e85-983f-29f9f16bb732 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.095518] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2116.095745] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-398b9cdb-233d-44fe-a316-1ca63500ac5a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.098200] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2116.098376] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2116.099340] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-21637a23-87e4-4d41-80e0-900d59b9e985 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.104908] env[63273]: DEBUG oslo_vmware.api [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for the task: (returnval){ [ 2116.104908] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5281049b-080b-bbcb-4205-7e91b3a41307" [ 2116.104908] env[63273]: _type = "Task" [ 2116.104908] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2116.113010] env[63273]: DEBUG oslo_vmware.api [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5281049b-080b-bbcb-4205-7e91b3a41307, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2116.463316] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2116.463316] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2116.463316] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Deleting the datastore file [datastore1] 0afc321d-108c-4c2e-81a7-664091127047 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2116.463316] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-fa549e6f-269c-4037-a01b-0bbe9f79a9aa {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.470557] env[63273]: DEBUG oslo_vmware.api [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 2116.470557] env[63273]: value = "task-5072194" [ 2116.470557] env[63273]: _type = "Task" [ 2116.470557] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2116.479816] env[63273]: DEBUG oslo_vmware.api [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': task-5072194, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2116.616099] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2116.616498] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Creating directory with path [datastore1] vmware_temp/844de822-f212-4244-9dc8-ba020879e4e6/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2116.616817] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-5b587217-e4ee-430e-9bcb-7dd9d85db007 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.628180] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Created directory with path [datastore1] vmware_temp/844de822-f212-4244-9dc8-ba020879e4e6/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2116.628365] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Fetch image to [datastore1] vmware_temp/844de822-f212-4244-9dc8-ba020879e4e6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2116.628537] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/844de822-f212-4244-9dc8-ba020879e4e6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2116.629276] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28c23e6b-1868-4699-8099-beaa52c75bdd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.636162] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2ccd8dc-788b-43a3-8233-435a3130641f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.646681] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f80775e-35ed-46cc-92e8-c24562d506fd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.678897] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3dec697-8118-42ab-bc88-bf107f6a63cb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.684852] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-8feb8afe-c340-433e-842a-6291fd3ac8e2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.705981] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2116.759087] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/844de822-f212-4244-9dc8-ba020879e4e6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2116.818088] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2116.818292] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/844de822-f212-4244-9dc8-ba020879e4e6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2116.980994] env[63273]: DEBUG oslo_vmware.api [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': task-5072194, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.083078} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2116.981311] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2116.981496] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2116.981673] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2116.981846] env[63273]: INFO nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Took 0.89 seconds to destroy the instance on the hypervisor. [ 2116.983979] env[63273]: DEBUG nova.compute.claims [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2116.984165] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2116.984419] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2117.144016] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ee2c8be-9252-42a3-a956-527803763bfd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2117.152183] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3719548e-6b9b-4c46-9e82-650153282c6e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2117.182140] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6b7f553-45e9-4aea-a718-28045dcf2699 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2117.190456] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f440db9f-d96e-4a84-b610-359e00abdba1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2117.204268] env[63273]: DEBUG nova.compute.provider_tree [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2117.213887] env[63273]: DEBUG nova.scheduler.client.report [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2117.229978] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.245s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2117.230570] env[63273]: ERROR nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2117.230570] env[63273]: Faults: ['InvalidArgument'] [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] Traceback (most recent call last): [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] self.driver.spawn(context, instance, image_meta, [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] self._fetch_image_if_missing(context, vi) [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] image_cache(vi, tmp_image_ds_loc) [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] vm_util.copy_virtual_disk( [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] session._wait_for_task(vmdk_copy_task) [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] return self.wait_for_task(task_ref) [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] return evt.wait() [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] result = hub.switch() [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] return self.greenlet.switch() [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] self.f(*self.args, **self.kw) [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] raise exceptions.translate_fault(task_info.error) [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] Faults: ['InvalidArgument'] [ 2117.230570] env[63273]: ERROR nova.compute.manager [instance: 0afc321d-108c-4c2e-81a7-664091127047] [ 2117.231574] env[63273]: DEBUG nova.compute.utils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2117.232812] env[63273]: DEBUG nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Build of instance 0afc321d-108c-4c2e-81a7-664091127047 was re-scheduled: A specified parameter was not correct: fileType [ 2117.232812] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 2117.233188] env[63273]: DEBUG nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 2117.233361] env[63273]: DEBUG nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 2117.233529] env[63273]: DEBUG nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2117.233690] env[63273]: DEBUG nova.network.neutron [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2117.568231] env[63273]: DEBUG nova.network.neutron [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2117.581256] env[63273]: INFO nova.compute.manager [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Took 0.35 seconds to deallocate network for instance. [ 2117.683449] env[63273]: INFO nova.scheduler.client.report [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Deleted allocations for instance 0afc321d-108c-4c2e-81a7-664091127047 [ 2117.726287] env[63273]: DEBUG oslo_concurrency.lockutils [None req-70c636b6-1767-497e-a28f-4de0fdfbeabb tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "0afc321d-108c-4c2e-81a7-664091127047" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 518.179s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2117.726636] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "0afc321d-108c-4c2e-81a7-664091127047" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 347.488s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2117.726877] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 0afc321d-108c-4c2e-81a7-664091127047] During sync_power_state the instance has a pending task (spawning). Skip. [ 2117.727122] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "0afc321d-108c-4c2e-81a7-664091127047" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2117.727748] env[63273]: DEBUG oslo_concurrency.lockutils [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "0afc321d-108c-4c2e-81a7-664091127047" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 322.119s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2117.728228] env[63273]: DEBUG oslo_concurrency.lockutils [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "0afc321d-108c-4c2e-81a7-664091127047-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2117.728570] env[63273]: DEBUG oslo_concurrency.lockutils [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "0afc321d-108c-4c2e-81a7-664091127047-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2117.728802] env[63273]: DEBUG oslo_concurrency.lockutils [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "0afc321d-108c-4c2e-81a7-664091127047-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2117.731051] env[63273]: INFO nova.compute.manager [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Terminating instance [ 2117.733541] env[63273]: DEBUG nova.compute.manager [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2117.733805] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2117.734431] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3c3e95a3-0fe3-4bae-91c0-836d5d5abeaf {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2117.745181] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12ae49b7-e5fd-48a5-b903-6d718f24d440 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2117.776119] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0afc321d-108c-4c2e-81a7-664091127047 could not be found. [ 2117.776333] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2117.776582] env[63273]: INFO nova.compute.manager [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2117.776858] env[63273]: DEBUG oslo.service.loopingcall [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2117.777119] env[63273]: DEBUG nova.compute.manager [-] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2117.777220] env[63273]: DEBUG nova.network.neutron [-] [instance: 0afc321d-108c-4c2e-81a7-664091127047] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2117.816876] env[63273]: DEBUG nova.network.neutron [-] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2117.825315] env[63273]: INFO nova.compute.manager [-] [instance: 0afc321d-108c-4c2e-81a7-664091127047] Took 0.05 seconds to deallocate network for instance. [ 2117.930642] env[63273]: DEBUG oslo_concurrency.lockutils [None req-91808c88-464c-4488-a9d6-5d6463256bc1 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "0afc321d-108c-4c2e-81a7-664091127047" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.203s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2139.891298] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2141.891397] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2142.891688] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2142.892193] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2142.904767] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2142.905012] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2142.905198] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2142.905357] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2142.906525] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6730bee1-41c6-446b-b96d-6011dbe10af3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2142.916015] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0593935b-47a9-42dc-a04d-8216dbe9e63b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2142.930529] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46a43a04-7424-41d1-abb3-b4f7482fff9e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2142.937427] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37216357-2661-4880-a4bd-ca44e03bdb5a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2142.967970] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180528MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2142.968185] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2142.968382] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2143.030177] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 2054144d-f068-4d52-b913-617b30d2d1c0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2143.030347] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2143.030476] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 886b6d46-98a1-4954-9113-0aadfd507d29 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2143.030597] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fd40e450-e51f-41a9-ba86-7b322f03969f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2143.030713] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 415fad0a-0af4-45b0-a958-ff900fbcb05b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2143.030836] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8afeca90-bf18-45ad-8b7e-d1a22632e0b0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2143.031026] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 6 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2143.031178] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1280MB phys_disk=100GB used_disk=6GB total_vcpus=48 used_vcpus=6 pci_stats=[] stats={'failed_builds': '86', 'num_instances': '6', 'num_vm_building': '6', 'num_task_deleting': '3', 'num_os_type_None': '6', 'num_proj_ff660b4621c14dc48d0e1fad4c290dcc': '2', 'io_workload': '6', 'num_proj_495ae26325184812a744fc95da67038b': '2', 'num_proj_5683e7f13d7540c1bda6b8e43f1428d0': '1', 'num_task_spawning': '3', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2143.118447] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a85678c7-5850-48fe-8fab-c0fe7956779b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2143.126357] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8993c0f1-d745-4af1-a52d-b3aeb74405d0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2143.156495] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46a52239-c19c-4e63-88e6-a257833b9f1d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2143.164478] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a034a2c4-d0a9-4f94-bc9f-179cc7acede3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2143.179291] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2143.187719] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2143.200930] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2143.201183] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.233s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2144.201333] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2144.201333] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 2144.201333] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 2144.218369] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2144.219082] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2144.219082] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2144.219082] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2144.219082] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2144.219082] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2144.219298] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 2144.219834] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2146.892120] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2146.892499] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2146.892499] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 2153.888606] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2163.387075] env[63273]: WARNING oslo_vmware.rw_handles [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2163.387075] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2163.387075] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2163.387075] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2163.387075] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2163.387075] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 2163.387075] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2163.387075] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2163.387075] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2163.387075] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2163.387075] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2163.387075] env[63273]: ERROR oslo_vmware.rw_handles [ 2163.387796] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/844de822-f212-4244-9dc8-ba020879e4e6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2163.389460] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2163.389742] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Copying Virtual Disk [datastore1] vmware_temp/844de822-f212-4244-9dc8-ba020879e4e6/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/844de822-f212-4244-9dc8-ba020879e4e6/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2163.390053] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-5328d624-6488-4118-9981-46c7c134c132 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.398799] env[63273]: DEBUG oslo_vmware.api [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for the task: (returnval){ [ 2163.398799] env[63273]: value = "task-5072195" [ 2163.398799] env[63273]: _type = "Task" [ 2163.398799] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2163.407219] env[63273]: DEBUG oslo_vmware.api [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Task: {'id': task-5072195, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2163.909332] env[63273]: DEBUG oslo_vmware.exceptions [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2163.909604] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2163.910171] env[63273]: ERROR nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2163.910171] env[63273]: Faults: ['InvalidArgument'] [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Traceback (most recent call last): [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] yield resources [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] self.driver.spawn(context, instance, image_meta, [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] self._fetch_image_if_missing(context, vi) [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] image_cache(vi, tmp_image_ds_loc) [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] vm_util.copy_virtual_disk( [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] session._wait_for_task(vmdk_copy_task) [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] return self.wait_for_task(task_ref) [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] return evt.wait() [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] result = hub.switch() [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] return self.greenlet.switch() [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] self.f(*self.args, **self.kw) [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] raise exceptions.translate_fault(task_info.error) [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Faults: ['InvalidArgument'] [ 2163.910171] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] [ 2163.911253] env[63273]: INFO nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Terminating instance [ 2163.912049] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2163.912263] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2163.912526] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-1101f8ed-71cd-4b4a-8f5f-c8208334687c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.914682] env[63273]: DEBUG nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2163.914882] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2163.915629] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd4aaf6c-c5d2-42b3-98a7-65ea98b43a00 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.922761] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2163.922962] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-af79b254-afe1-4697-92fa-e7af730d5c2f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.925214] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2163.925392] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2163.926354] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5cef6e64-ec1d-4cab-a75d-fe8d015f3af1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.931236] env[63273]: DEBUG oslo_vmware.api [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 2163.931236] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52190735-af7a-18f8-923c-390a3e148b81" [ 2163.931236] env[63273]: _type = "Task" [ 2163.931236] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2163.940827] env[63273]: DEBUG oslo_vmware.api [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52190735-af7a-18f8-923c-390a3e148b81, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2164.001079] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2164.001079] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2164.001345] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Deleting the datastore file [datastore1] 2054144d-f068-4d52-b913-617b30d2d1c0 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2164.001441] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-d88b4ad8-c1a6-48bf-a6c7-77204f2a55e1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.009704] env[63273]: DEBUG oslo_vmware.api [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for the task: (returnval){ [ 2164.009704] env[63273]: value = "task-5072197" [ 2164.009704] env[63273]: _type = "Task" [ 2164.009704] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2164.018362] env[63273]: DEBUG oslo_vmware.api [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Task: {'id': task-5072197, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2164.441678] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2164.442126] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating directory with path [datastore1] vmware_temp/149dcf84-938d-458d-99dd-0ce4c2a3dd08/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2164.442126] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-d4516758-b04c-4a44-be83-94878de5173c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.453970] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created directory with path [datastore1] vmware_temp/149dcf84-938d-458d-99dd-0ce4c2a3dd08/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2164.454188] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Fetch image to [datastore1] vmware_temp/149dcf84-938d-458d-99dd-0ce4c2a3dd08/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2164.454389] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/149dcf84-938d-458d-99dd-0ce4c2a3dd08/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2164.455179] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8633ff4-5258-43f6-9dfb-249652db1f89 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.461969] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-01d7b8ca-92a5-4b92-a4d2-fc2398d36782 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.471258] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77f69496-9332-4469-aa93-925c0039c13b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.502681] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-298d5040-9f6f-4a91-a7b3-e1a6a04a2be3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.508744] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-84a838ec-10ec-4a22-ba9c-9636132791f3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.518256] env[63273]: DEBUG oslo_vmware.api [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Task: {'id': task-5072197, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.070369} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2164.518539] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2164.518724] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2164.518894] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2164.519081] env[63273]: INFO nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2164.521332] env[63273]: DEBUG nova.compute.claims [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2164.521503] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2164.521747] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2164.537368] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2164.593680] env[63273]: DEBUG oslo_vmware.rw_handles [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/149dcf84-938d-458d-99dd-0ce4c2a3dd08/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2164.652331] env[63273]: DEBUG oslo_vmware.rw_handles [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2164.652531] env[63273]: DEBUG oslo_vmware.rw_handles [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/149dcf84-938d-458d-99dd-0ce4c2a3dd08/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2164.704451] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-086a74fb-3b41-479e-afe6-2b002d771435 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.711989] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b6937e2-165f-42ee-bacc-849c36d4a42c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.740957] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e82c5dac-d42e-4257-a2e5-9e1e934ce97e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.748823] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77b3bd97-d3ee-402c-9ed6-00bf7dbae808 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.763800] env[63273]: DEBUG nova.compute.provider_tree [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2164.774047] env[63273]: DEBUG nova.scheduler.client.report [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2164.789475] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.268s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2164.790063] env[63273]: ERROR nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2164.790063] env[63273]: Faults: ['InvalidArgument'] [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Traceback (most recent call last): [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] self.driver.spawn(context, instance, image_meta, [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] self._fetch_image_if_missing(context, vi) [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] image_cache(vi, tmp_image_ds_loc) [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] vm_util.copy_virtual_disk( [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] session._wait_for_task(vmdk_copy_task) [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] return self.wait_for_task(task_ref) [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] return evt.wait() [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] result = hub.switch() [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] return self.greenlet.switch() [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] self.f(*self.args, **self.kw) [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] raise exceptions.translate_fault(task_info.error) [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Faults: ['InvalidArgument'] [ 2164.790063] env[63273]: ERROR nova.compute.manager [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] [ 2164.790866] env[63273]: DEBUG nova.compute.utils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2164.792213] env[63273]: DEBUG nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Build of instance 2054144d-f068-4d52-b913-617b30d2d1c0 was re-scheduled: A specified parameter was not correct: fileType [ 2164.792213] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 2164.792633] env[63273]: DEBUG nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 2164.792808] env[63273]: DEBUG nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 2164.792975] env[63273]: DEBUG nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2164.793150] env[63273]: DEBUG nova.network.neutron [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2165.220705] env[63273]: DEBUG nova.network.neutron [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2165.231809] env[63273]: INFO nova.compute.manager [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Took 0.44 seconds to deallocate network for instance. [ 2165.333882] env[63273]: INFO nova.scheduler.client.report [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Deleted allocations for instance 2054144d-f068-4d52-b913-617b30d2d1c0 [ 2165.357312] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f53aee44-680e-4ad7-9b6b-f73914644ebc tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "2054144d-f068-4d52-b913-617b30d2d1c0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 530.362s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2165.357574] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "2054144d-f068-4d52-b913-617b30d2d1c0" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 395.119s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2165.357762] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] During sync_power_state the instance has a pending task (spawning). Skip. [ 2165.357934] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "2054144d-f068-4d52-b913-617b30d2d1c0" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2165.358460] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "2054144d-f068-4d52-b913-617b30d2d1c0" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 334.473s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2165.358681] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "2054144d-f068-4d52-b913-617b30d2d1c0-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2165.358884] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "2054144d-f068-4d52-b913-617b30d2d1c0-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2165.359104] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "2054144d-f068-4d52-b913-617b30d2d1c0-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2165.361454] env[63273]: INFO nova.compute.manager [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Terminating instance [ 2165.363569] env[63273]: DEBUG nova.compute.manager [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2165.363569] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2165.363680] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4650241e-d091-4449-9abb-b878df882a93 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2165.373031] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1576f5d-20b8-4304-842e-f2a9ec3bfab9 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2165.405024] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 2054144d-f068-4d52-b913-617b30d2d1c0 could not be found. [ 2165.405235] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2165.405412] env[63273]: INFO nova.compute.manager [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2165.405652] env[63273]: DEBUG oslo.service.loopingcall [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2165.405871] env[63273]: DEBUG nova.compute.manager [-] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2165.405965] env[63273]: DEBUG nova.network.neutron [-] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2165.429962] env[63273]: DEBUG nova.network.neutron [-] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2165.437811] env[63273]: INFO nova.compute.manager [-] [instance: 2054144d-f068-4d52-b913-617b30d2d1c0] Took 0.03 seconds to deallocate network for instance. [ 2165.524693] env[63273]: DEBUG oslo_concurrency.lockutils [None req-1b5326ae-1f29-4970-8020-7a2704243646 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "2054144d-f068-4d52-b913-617b30d2d1c0" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.166s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2187.603695] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "fd40e450-e51f-41a9-ba86-7b322f03969f" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2200.891645] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2202.893198] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2203.892029] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2203.892323] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2204.887112] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2204.912709] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2204.913226] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 2204.913586] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 2204.929825] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2204.930014] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2204.930160] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2204.930355] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2204.930515] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2204.930672] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 2204.931527] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2204.944598] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2204.944808] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2204.944976] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2204.945138] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2204.946252] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7265464f-7036-4d82-9b28-b84081ac2a81 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.955640] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7502a29-1170-48a1-86ff-0b931aaf5090 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.970919] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b950d3c-f381-4079-9aa7-7ef36b8793ce {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.977922] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b209a58-aff1-4348-ad2b-9432b742da7a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2205.008068] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180517MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2205.008245] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2205.008417] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2205.066638] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2205.066802] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 886b6d46-98a1-4954-9113-0aadfd507d29 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2205.066935] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fd40e450-e51f-41a9-ba86-7b322f03969f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2205.067075] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 415fad0a-0af4-45b0-a958-ff900fbcb05b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2205.067202] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8afeca90-bf18-45ad-8b7e-d1a22632e0b0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2205.067390] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 5 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2205.067583] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1152MB phys_disk=100GB used_disk=5GB total_vcpus=48 used_vcpus=5 pci_stats=[] stats={'failed_builds': '87', 'num_instances': '5', 'num_vm_building': '5', 'num_task_deleting': '3', 'num_os_type_None': '5', 'num_proj_495ae26325184812a744fc95da67038b': '2', 'io_workload': '5', 'num_proj_5683e7f13d7540c1bda6b8e43f1428d0': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1', 'num_task_spawning': '2', 'num_proj_ff660b4621c14dc48d0e1fad4c290dcc': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2205.137511] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0b258f7-93df-47b1-81a9-467ff25a9ea4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2205.145260] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55e4c62e-3ab1-4d2a-94fc-6c0ff8e72055 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2205.174922] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e770b099-4d5f-458c-ba6d-81c9544ce272 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2205.183070] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7c7b625-6793-4580-b3ce-96511e71ce12 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2205.196686] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2205.205251] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2205.219639] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2205.219849] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.211s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2207.181139] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2207.181567] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 2208.892631] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2210.588057] env[63273]: WARNING oslo_vmware.rw_handles [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2210.588057] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2210.588057] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2210.588057] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2210.588057] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2210.588057] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 2210.588057] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2210.588057] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2210.588057] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2210.588057] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2210.588057] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2210.588057] env[63273]: ERROR oslo_vmware.rw_handles [ 2210.588057] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/149dcf84-938d-458d-99dd-0ce4c2a3dd08/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2210.590039] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2210.590287] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Copying Virtual Disk [datastore1] vmware_temp/149dcf84-938d-458d-99dd-0ce4c2a3dd08/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/149dcf84-938d-458d-99dd-0ce4c2a3dd08/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2210.590577] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-5c6ace3b-8da0-4260-bf02-107ca8c6ce55 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2210.598866] env[63273]: DEBUG oslo_vmware.api [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 2210.598866] env[63273]: value = "task-5072198" [ 2210.598866] env[63273]: _type = "Task" [ 2210.598866] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2210.607699] env[63273]: DEBUG oslo_vmware.api [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072198, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2211.109982] env[63273]: DEBUG oslo_vmware.exceptions [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2211.110300] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2211.110859] env[63273]: ERROR nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2211.110859] env[63273]: Faults: ['InvalidArgument'] [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Traceback (most recent call last): [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] yield resources [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] self.driver.spawn(context, instance, image_meta, [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] self._fetch_image_if_missing(context, vi) [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] image_cache(vi, tmp_image_ds_loc) [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] vm_util.copy_virtual_disk( [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] session._wait_for_task(vmdk_copy_task) [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] return self.wait_for_task(task_ref) [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] return evt.wait() [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] result = hub.switch() [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] return self.greenlet.switch() [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] self.f(*self.args, **self.kw) [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] raise exceptions.translate_fault(task_info.error) [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Faults: ['InvalidArgument'] [ 2211.110859] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] [ 2211.111895] env[63273]: INFO nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Terminating instance [ 2211.112922] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2211.113151] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2211.113397] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-5c5e51c7-c0e8-4f21-98db-1c81398e0fdd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.116891] env[63273]: DEBUG nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2211.117097] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2211.117919] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-569241da-828e-4c88-841a-b33f58c8accc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.125483] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2211.125790] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-a0491a2a-58bd-4264-a860-ccde09251267 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.128224] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2211.128397] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2211.129378] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c8fad6c4-0efe-4f91-87ae-f05d38c85475 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.134454] env[63273]: DEBUG oslo_vmware.api [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for the task: (returnval){ [ 2211.134454] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]5209c074-9bb2-ae54-6abe-6ac4ae5a47c3" [ 2211.134454] env[63273]: _type = "Task" [ 2211.134454] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2211.142428] env[63273]: DEBUG oslo_vmware.api [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]5209c074-9bb2-ae54-6abe-6ac4ae5a47c3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2211.195967] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2211.196241] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2211.196419] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleting the datastore file [datastore1] 7262f23a-8ee4-4468-8e49-2e9297352dc9 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2211.196788] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-231e4448-3ed7-4956-8e39-6ac70e87cb13 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.204550] env[63273]: DEBUG oslo_vmware.api [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 2211.204550] env[63273]: value = "task-5072200" [ 2211.204550] env[63273]: _type = "Task" [ 2211.204550] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2211.213283] env[63273]: DEBUG oslo_vmware.api [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072200, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2211.645108] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2211.645502] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Creating directory with path [datastore1] vmware_temp/ea7f86e1-73d3-484c-bff4-f96edf555533/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2211.645628] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e299f247-138a-41a5-8a88-77ebfcd9f767 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.657616] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Created directory with path [datastore1] vmware_temp/ea7f86e1-73d3-484c-bff4-f96edf555533/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2211.657813] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Fetch image to [datastore1] vmware_temp/ea7f86e1-73d3-484c-bff4-f96edf555533/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2211.657988] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/ea7f86e1-73d3-484c-bff4-f96edf555533/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2211.658780] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9dabc79e-64ee-47b5-aeb5-bacdf4ed8657 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.665545] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3bc15e6c-5e37-4816-843d-30463ad9fb93 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.674892] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bac363de-822e-4f99-8aa4-a1b1d6a110bd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.708641] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5bb887e6-7586-4698-b921-d25e6d78fd6f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.717435] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-096c3075-c8e9-4b65-b449-018e7487f5b2 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.719260] env[63273]: DEBUG oslo_vmware.api [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': task-5072200, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.086153} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2211.719508] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2211.719688] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2211.719860] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2211.720043] env[63273]: INFO nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2211.722644] env[63273]: DEBUG nova.compute.claims [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2211.722894] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2211.723089] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2211.742455] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2211.802518] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ea7f86e1-73d3-484c-bff4-f96edf555533/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2211.866141] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2211.866344] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ea7f86e1-73d3-484c-bff4-f96edf555533/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2211.901903] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08c89a78-f280-4f08-809e-22276930453f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.909567] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc194443-0afd-44e5-ae99-bcd2527818a7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.939661] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a295badc-b3d2-49be-9678-e755e1cfd9b4 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.947302] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7f09c0d-b060-41bf-8e77-be366ac080cc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.961150] env[63273]: DEBUG nova.compute.provider_tree [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2211.971456] env[63273]: DEBUG nova.scheduler.client.report [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2211.985951] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.263s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2211.986518] env[63273]: ERROR nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2211.986518] env[63273]: Faults: ['InvalidArgument'] [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Traceback (most recent call last): [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] self.driver.spawn(context, instance, image_meta, [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] self._fetch_image_if_missing(context, vi) [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] image_cache(vi, tmp_image_ds_loc) [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] vm_util.copy_virtual_disk( [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] session._wait_for_task(vmdk_copy_task) [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] return self.wait_for_task(task_ref) [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] return evt.wait() [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] result = hub.switch() [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] return self.greenlet.switch() [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] self.f(*self.args, **self.kw) [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] raise exceptions.translate_fault(task_info.error) [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Faults: ['InvalidArgument'] [ 2211.986518] env[63273]: ERROR nova.compute.manager [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] [ 2211.987391] env[63273]: DEBUG nova.compute.utils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2211.988932] env[63273]: DEBUG nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Build of instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 was re-scheduled: A specified parameter was not correct: fileType [ 2211.988932] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 2211.989333] env[63273]: DEBUG nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 2211.989505] env[63273]: DEBUG nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 2211.989676] env[63273]: DEBUG nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2211.989840] env[63273]: DEBUG nova.network.neutron [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2212.327405] env[63273]: DEBUG nova.network.neutron [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2212.341298] env[63273]: INFO nova.compute.manager [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Took 0.35 seconds to deallocate network for instance. [ 2212.440521] env[63273]: INFO nova.scheduler.client.report [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Deleted allocations for instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 [ 2212.468540] env[63273]: DEBUG oslo_concurrency.lockutils [None req-d57e5aed-4b19-42fb-b38b-ee56165eee23 tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "7262f23a-8ee4-4468-8e49-2e9297352dc9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 528.383s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2212.468540] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "7262f23a-8ee4-4468-8e49-2e9297352dc9" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 442.229s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2212.468540] env[63273]: INFO nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] During sync_power_state the instance has a pending task (spawning). Skip. [ 2212.468540] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "7262f23a-8ee4-4468-8e49-2e9297352dc9" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2212.468540] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "7262f23a-8ee4-4468-8e49-2e9297352dc9" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 332.392s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2212.468911] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "7262f23a-8ee4-4468-8e49-2e9297352dc9-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2212.469025] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "7262f23a-8ee4-4468-8e49-2e9297352dc9-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2212.469473] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "7262f23a-8ee4-4468-8e49-2e9297352dc9-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2212.472172] env[63273]: INFO nova.compute.manager [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Terminating instance [ 2212.474511] env[63273]: DEBUG nova.compute.manager [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2212.474750] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2212.475430] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d4393e4d-5cb6-4e80-9118-7220dc35b372 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2212.486473] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7901fed-907a-40c4-b8cf-10aa0011ae18 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2212.522638] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7262f23a-8ee4-4468-8e49-2e9297352dc9 could not be found. [ 2212.522863] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2212.523064] env[63273]: INFO nova.compute.manager [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Took 0.05 seconds to destroy the instance on the hypervisor. [ 2212.523320] env[63273]: DEBUG oslo.service.loopingcall [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2212.523566] env[63273]: DEBUG nova.compute.manager [-] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2212.523664] env[63273]: DEBUG nova.network.neutron [-] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2212.550101] env[63273]: DEBUG nova.network.neutron [-] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2212.559840] env[63273]: INFO nova.compute.manager [-] [instance: 7262f23a-8ee4-4468-8e49-2e9297352dc9] Took 0.04 seconds to deallocate network for instance. [ 2212.655302] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2008e3ac-cd91-440d-84c1-d0826d69405a tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Lock "7262f23a-8ee4-4468-8e49-2e9297352dc9" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.187s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2215.886501] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2224.760081] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "415fad0a-0af4-45b0-a958-ff900fbcb05b" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2260.587469] env[63273]: WARNING oslo_vmware.rw_handles [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2260.587469] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2260.587469] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2260.587469] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2260.587469] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2260.587469] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 2260.587469] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2260.587469] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2260.587469] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2260.587469] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2260.587469] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2260.587469] env[63273]: ERROR oslo_vmware.rw_handles [ 2260.588550] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/ea7f86e1-73d3-484c-bff4-f96edf555533/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2260.591018] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2260.591688] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Copying Virtual Disk [datastore1] vmware_temp/ea7f86e1-73d3-484c-bff4-f96edf555533/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/ea7f86e1-73d3-484c-bff4-f96edf555533/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2260.591822] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-a90a1b81-08ce-4854-b131-0618ff56edf5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2260.600613] env[63273]: DEBUG oslo_vmware.api [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for the task: (returnval){ [ 2260.600613] env[63273]: value = "task-5072201" [ 2260.600613] env[63273]: _type = "Task" [ 2260.600613] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2260.610370] env[63273]: DEBUG oslo_vmware.api [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Task: {'id': task-5072201, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2261.111799] env[63273]: DEBUG oslo_vmware.exceptions [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2261.112549] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2261.113048] env[63273]: ERROR nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2261.113048] env[63273]: Faults: ['InvalidArgument'] [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Traceback (most recent call last): [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] yield resources [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] self.driver.spawn(context, instance, image_meta, [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] self._fetch_image_if_missing(context, vi) [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] image_cache(vi, tmp_image_ds_loc) [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] vm_util.copy_virtual_disk( [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] session._wait_for_task(vmdk_copy_task) [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] return self.wait_for_task(task_ref) [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] return evt.wait() [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] result = hub.switch() [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] return self.greenlet.switch() [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] self.f(*self.args, **self.kw) [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] raise exceptions.translate_fault(task_info.error) [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Faults: ['InvalidArgument'] [ 2261.113048] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] [ 2261.114080] env[63273]: INFO nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Terminating instance [ 2261.114983] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2261.115601] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2261.115601] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-70019142-86bf-4a0a-a3ee-ec02401c3fbd {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.118118] env[63273]: DEBUG nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2261.118470] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2261.119327] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a90f495-ad9f-4d41-a93e-d41c464571ae {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.127018] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2261.127018] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-4b1c902f-aa5f-460f-9682-e68398bc82e0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.128799] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2261.129150] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2261.130264] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2db35c8f-b0df-45f6-9c3f-7e95a93c765b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.135701] env[63273]: DEBUG oslo_vmware.api [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 2261.135701] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ee418a-fa2d-23c7-6cb8-b4642fdb0b19" [ 2261.135701] env[63273]: _type = "Task" [ 2261.135701] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2261.143728] env[63273]: DEBUG oslo_vmware.api [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52ee418a-fa2d-23c7-6cb8-b4642fdb0b19, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2261.200534] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2261.202029] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2261.202029] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Deleting the datastore file [datastore1] 886b6d46-98a1-4954-9113-0aadfd507d29 {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2261.202029] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-270f6ac9-d9ac-46dc-97f1-1cdd390d2bf3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.208254] env[63273]: DEBUG oslo_vmware.api [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for the task: (returnval){ [ 2261.208254] env[63273]: value = "task-5072203" [ 2261.208254] env[63273]: _type = "Task" [ 2261.208254] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2261.217661] env[63273]: DEBUG oslo_vmware.api [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Task: {'id': task-5072203, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2261.646625] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2261.646890] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Creating directory with path [datastore1] vmware_temp/ad074538-e8b2-4a83-931b-3c76bd22bffc/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2261.647141] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-14c362fb-2390-44f8-b5b8-5f6283e74058 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.659219] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Created directory with path [datastore1] vmware_temp/ad074538-e8b2-4a83-931b-3c76bd22bffc/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2261.659433] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Fetch image to [datastore1] vmware_temp/ad074538-e8b2-4a83-931b-3c76bd22bffc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2261.659603] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/ad074538-e8b2-4a83-931b-3c76bd22bffc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2261.660388] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe59a6f7-15ed-4445-9eb1-6c9317fec35a {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.667848] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd1b52af-30f3-42a4-b043-cb7cb1eb7619 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.677160] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d052eba-9b8b-4607-b9ef-2b0186311e71 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.708023] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb89474e-e228-4367-bb14-e1a2ef150842 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.718551] env[63273]: DEBUG oslo_vmware.api [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Task: {'id': task-5072203, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.068996} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2261.720039] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2261.720236] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2261.720413] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2261.720591] env[63273]: INFO nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2261.722738] env[63273]: DEBUG nova.compute.claims [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2261.722974] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2261.723143] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2261.725815] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-ce9b5fa1-f007-4598-8da4-5cd1e9ec5991 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.750734] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2261.812629] env[63273]: DEBUG oslo_vmware.rw_handles [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ad074538-e8b2-4a83-931b-3c76bd22bffc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2261.874564] env[63273]: DEBUG oslo_vmware.rw_handles [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2261.874766] env[63273]: DEBUG oslo_vmware.rw_handles [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ad074538-e8b2-4a83-931b-3c76bd22bffc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2261.915550] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9b3d02c-5c16-4a59-bdb4-f565bf1ce1f7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.923425] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39e3116c-adeb-4ba6-ace7-d24e8555a95c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.953020] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bee9464b-1e05-45dd-8f79-150dc214e3e5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.961156] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19a33acb-e81e-4016-83a0-22df98284e38 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.975019] env[63273]: DEBUG nova.compute.provider_tree [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2261.985393] env[63273]: DEBUG nova.scheduler.client.report [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2261.999258] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.276s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2261.999921] env[63273]: ERROR nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2261.999921] env[63273]: Faults: ['InvalidArgument'] [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Traceback (most recent call last): [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] self.driver.spawn(context, instance, image_meta, [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] self._fetch_image_if_missing(context, vi) [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] image_cache(vi, tmp_image_ds_loc) [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] vm_util.copy_virtual_disk( [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] session._wait_for_task(vmdk_copy_task) [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] return self.wait_for_task(task_ref) [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] return evt.wait() [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] result = hub.switch() [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] return self.greenlet.switch() [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] self.f(*self.args, **self.kw) [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] raise exceptions.translate_fault(task_info.error) [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Faults: ['InvalidArgument'] [ 2261.999921] env[63273]: ERROR nova.compute.manager [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] [ 2262.000846] env[63273]: DEBUG nova.compute.utils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2262.002645] env[63273]: DEBUG nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Build of instance 886b6d46-98a1-4954-9113-0aadfd507d29 was re-scheduled: A specified parameter was not correct: fileType [ 2262.002645] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 2262.003135] env[63273]: DEBUG nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 2262.003408] env[63273]: DEBUG nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 2262.003682] env[63273]: DEBUG nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2262.003938] env[63273]: DEBUG nova.network.neutron [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2262.347346] env[63273]: DEBUG nova.network.neutron [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2262.363059] env[63273]: INFO nova.compute.manager [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Took 0.36 seconds to deallocate network for instance. [ 2262.466710] env[63273]: INFO nova.scheduler.client.report [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Deleted allocations for instance 886b6d46-98a1-4954-9113-0aadfd507d29 [ 2262.488303] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f3f4da9f-dd9c-4800-b626-60a4c4395f5b tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "886b6d46-98a1-4954-9113-0aadfd507d29" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 452.129s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2262.488460] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "886b6d46-98a1-4954-9113-0aadfd507d29" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 256.853s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2262.488667] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Acquiring lock "886b6d46-98a1-4954-9113-0aadfd507d29-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2262.488857] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "886b6d46-98a1-4954-9113-0aadfd507d29-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2262.489053] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "886b6d46-98a1-4954-9113-0aadfd507d29-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2262.491266] env[63273]: INFO nova.compute.manager [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Terminating instance [ 2262.493112] env[63273]: DEBUG nova.compute.manager [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2262.493253] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2262.493782] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e5bd345d-877c-49fa-b22e-6d056f701eb8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2262.506558] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5891b040-259b-439b-ba39-498fec253be7 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2262.534674] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 886b6d46-98a1-4954-9113-0aadfd507d29 could not be found. [ 2262.534945] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2262.535150] env[63273]: INFO nova.compute.manager [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2262.535405] env[63273]: DEBUG oslo.service.loopingcall [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2262.535633] env[63273]: DEBUG nova.compute.manager [-] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2262.535730] env[63273]: DEBUG nova.network.neutron [-] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2262.561996] env[63273]: DEBUG nova.network.neutron [-] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2262.571920] env[63273]: INFO nova.compute.manager [-] [instance: 886b6d46-98a1-4954-9113-0aadfd507d29] Took 0.04 seconds to deallocate network for instance. [ 2262.656928] env[63273]: DEBUG oslo_concurrency.lockutils [None req-4e12fb5d-417c-4e65-9d27-026810da07d3 tempest-AttachVolumeNegativeTest-715869468 tempest-AttachVolumeNegativeTest-715869468-project-member] Lock "886b6d46-98a1-4954-9113-0aadfd507d29" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.168s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2262.891478] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2263.891742] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2264.891618] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2264.891847] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2264.892138] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2264.903587] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2264.903809] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2264.903980] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2264.904151] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2264.905287] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1dc89171-6219-4390-bb44-5e5f2bf3fcf8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2264.914472] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef28cbd0-695c-444d-9481-81e802183255 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2264.929458] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9f27dd2-1c4f-4c8d-a78d-3530aea849fe {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2264.936747] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de0e560e-3ab6-41c2-a671-f02150e78c53 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2264.966874] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180518MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2264.967078] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2264.967153] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2265.023321] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance fd40e450-e51f-41a9-ba86-7b322f03969f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2265.023489] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 415fad0a-0af4-45b0-a958-ff900fbcb05b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2265.023619] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8afeca90-bf18-45ad-8b7e-d1a22632e0b0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2265.023802] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 3 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2265.023955] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=896MB phys_disk=100GB used_disk=3GB total_vcpus=48 used_vcpus=3 pci_stats=[] stats={'failed_builds': '89', 'num_instances': '3', 'num_vm_building': '3', 'num_task_deleting': '2', 'num_os_type_None': '3', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1', 'io_workload': '3', 'num_proj_ff660b4621c14dc48d0e1fad4c290dcc': '1', 'num_task_spawning': '1', 'num_proj_495ae26325184812a744fc95da67038b': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2265.078356] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6abae95b-a717-49e7-af4d-17fbf99cf90e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2265.086573] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c97ef9d-f643-4d27-87c9-672eb6dad385 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2265.116448] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22388497-20c6-4bf2-876a-001607a4b9e5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2265.125454] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c82f5b2-5c41-4e98-a4e7-2c45c2d06339 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2265.140323] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2265.148740] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2265.164601] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2265.164794] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.198s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2266.164896] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2266.165184] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 2266.165184] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 2266.180097] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2266.180251] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2266.180388] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2266.180520] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 2266.891704] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2266.891704] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 2269.893934] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2272.007696] env[63273]: DEBUG oslo_concurrency.lockutils [None req-e3ae35b9-2cd3-431f-8e41-4a780ece842c tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquiring lock "8afeca90-bf18-45ad-8b7e-d1a22632e0b0" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2277.886574] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2311.298069] env[63273]: WARNING oslo_vmware.rw_handles [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2311.298069] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2311.298069] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2311.298069] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2311.298069] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2311.298069] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 2311.298069] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2311.298069] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2311.298069] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2311.298069] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2311.298069] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2311.298069] env[63273]: ERROR oslo_vmware.rw_handles [ 2311.298752] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/ad074538-e8b2-4a83-931b-3c76bd22bffc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2311.300374] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2311.300636] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Copying Virtual Disk [datastore1] vmware_temp/ad074538-e8b2-4a83-931b-3c76bd22bffc/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/ad074538-e8b2-4a83-931b-3c76bd22bffc/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2311.300940] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-66225348-5f6a-48bc-a3b7-b7cc703a1569 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2311.310980] env[63273]: DEBUG oslo_vmware.api [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 2311.310980] env[63273]: value = "task-5072204" [ 2311.310980] env[63273]: _type = "Task" [ 2311.310980] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2311.319440] env[63273]: DEBUG oslo_vmware.api [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': task-5072204, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2311.821328] env[63273]: DEBUG oslo_vmware.exceptions [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2311.821607] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2311.822170] env[63273]: ERROR nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2311.822170] env[63273]: Faults: ['InvalidArgument'] [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Traceback (most recent call last): [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] yield resources [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] self.driver.spawn(context, instance, image_meta, [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] self._fetch_image_if_missing(context, vi) [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] image_cache(vi, tmp_image_ds_loc) [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] vm_util.copy_virtual_disk( [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] session._wait_for_task(vmdk_copy_task) [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] return self.wait_for_task(task_ref) [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] return evt.wait() [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] result = hub.switch() [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] return self.greenlet.switch() [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] self.f(*self.args, **self.kw) [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] raise exceptions.translate_fault(task_info.error) [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Faults: ['InvalidArgument'] [ 2311.822170] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] [ 2311.823022] env[63273]: INFO nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Terminating instance [ 2311.824074] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2311.825244] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2311.825874] env[63273]: DEBUG nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2311.826090] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2311.826340] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0f5f050e-3b5e-4764-b728-e52211b3ef66 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2311.828967] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1495e94-9a76-4c50-80a2-5e0e75bb39ab {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2311.836328] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2311.836565] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-dfe3e792-510f-40e0-8355-ebffdbd7854d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2311.838975] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2311.839171] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2311.840159] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f9f87d84-9fcb-4b55-9aa2-30d490b7a903 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2311.845383] env[63273]: DEBUG oslo_vmware.api [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for the task: (returnval){ [ 2311.845383] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e7784d-c15f-86bb-a605-b9270fc79ff1" [ 2311.845383] env[63273]: _type = "Task" [ 2311.845383] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2311.854267] env[63273]: DEBUG oslo_vmware.api [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]52e7784d-c15f-86bb-a605-b9270fc79ff1, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2311.915063] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2311.915063] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2311.915312] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Deleting the datastore file [datastore1] fd40e450-e51f-41a9-ba86-7b322f03969f {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2311.915717] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-0b69424f-ae52-42e8-b4e6-936a051d0071 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2311.924982] env[63273]: DEBUG oslo_vmware.api [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 2311.924982] env[63273]: value = "task-5072206" [ 2311.924982] env[63273]: _type = "Task" [ 2311.924982] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2311.937667] env[63273]: DEBUG oslo_vmware.api [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': task-5072206, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2312.355681] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2312.356048] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Creating directory with path [datastore1] vmware_temp/10339c8c-d99a-49f8-9c0d-60e00698dcb4/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2312.356196] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-95897150-c362-4df2-93a6-55b91b114ccb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2312.367569] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Created directory with path [datastore1] vmware_temp/10339c8c-d99a-49f8-9c0d-60e00698dcb4/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2312.367755] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Fetch image to [datastore1] vmware_temp/10339c8c-d99a-49f8-9c0d-60e00698dcb4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2312.367920] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/10339c8c-d99a-49f8-9c0d-60e00698dcb4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2312.368648] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-683a7a94-df23-4b81-b364-3426ccaf0953 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2312.375343] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ee12fd1-8c03-42d6-b2c1-e33995641bda {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2312.384236] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d550de00-c1f6-4a09-a6da-2f6e91fa710b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2312.414333] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57a4c319-ae12-4bcc-8d9d-7ba86cbfefce {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2312.420217] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-0da6edbe-1fcf-45a1-8759-b115867e8ef8 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2312.433903] env[63273]: DEBUG oslo_vmware.api [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': task-5072206, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.09438} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2312.434293] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2312.434506] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2312.434715] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2312.434928] env[63273]: INFO nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Took 0.61 seconds to destroy the instance on the hypervisor. [ 2312.437117] env[63273]: DEBUG nova.compute.claims [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2312.437296] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2312.437553] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2312.443807] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2312.499259] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/10339c8c-d99a-49f8-9c0d-60e00698dcb4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2312.560659] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2312.561280] env[63273]: DEBUG oslo_vmware.rw_handles [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/10339c8c-d99a-49f8-9c0d-60e00698dcb4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2312.593385] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-289adc85-ff18-40bd-8192-007ee83c045e {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2312.601217] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-188838d6-b618-48ef-b492-b56ba405a5ae {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2312.634304] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1163cd68-f2ee-4c7c-8e60-fbe5327d6c69 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2312.642203] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78c3f3e6-4145-4158-a4d7-e1a310ca753b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2312.655885] env[63273]: DEBUG nova.compute.provider_tree [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2312.664793] env[63273]: DEBUG nova.scheduler.client.report [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2312.678595] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.241s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2312.679163] env[63273]: ERROR nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2312.679163] env[63273]: Faults: ['InvalidArgument'] [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Traceback (most recent call last): [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] self.driver.spawn(context, instance, image_meta, [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] self._fetch_image_if_missing(context, vi) [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] image_cache(vi, tmp_image_ds_loc) [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] vm_util.copy_virtual_disk( [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] session._wait_for_task(vmdk_copy_task) [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] return self.wait_for_task(task_ref) [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] return evt.wait() [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] result = hub.switch() [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] return self.greenlet.switch() [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] self.f(*self.args, **self.kw) [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] raise exceptions.translate_fault(task_info.error) [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Faults: ['InvalidArgument'] [ 2312.679163] env[63273]: ERROR nova.compute.manager [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] [ 2312.680093] env[63273]: DEBUG nova.compute.utils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2312.681318] env[63273]: DEBUG nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Build of instance fd40e450-e51f-41a9-ba86-7b322f03969f was re-scheduled: A specified parameter was not correct: fileType [ 2312.681318] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 2312.681694] env[63273]: DEBUG nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 2312.681871] env[63273]: DEBUG nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 2312.682058] env[63273]: DEBUG nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2312.682230] env[63273]: DEBUG nova.network.neutron [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2313.269484] env[63273]: DEBUG nova.network.neutron [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2313.286121] env[63273]: INFO nova.compute.manager [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Took 0.60 seconds to deallocate network for instance. [ 2313.395676] env[63273]: INFO nova.scheduler.client.report [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Deleted allocations for instance fd40e450-e51f-41a9-ba86-7b322f03969f [ 2313.420504] env[63273]: DEBUG oslo_concurrency.lockutils [None req-738c094d-193e-4912-956b-87a25e340ec9 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fd40e450-e51f-41a9-ba86-7b322f03969f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 321.182s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2313.420771] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fd40e450-e51f-41a9-ba86-7b322f03969f" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 125.817s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2313.420989] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "fd40e450-e51f-41a9-ba86-7b322f03969f-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2313.421249] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fd40e450-e51f-41a9-ba86-7b322f03969f-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2313.421430] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fd40e450-e51f-41a9-ba86-7b322f03969f-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2313.423445] env[63273]: INFO nova.compute.manager [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Terminating instance [ 2313.425160] env[63273]: DEBUG nova.compute.manager [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2313.425357] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2313.425826] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f4db3766-6101-440b-9271-11c7937c8b61 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2313.437073] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea1bfdad-86f0-44d9-a246-9376516c2a8b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2313.465749] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fd40e450-e51f-41a9-ba86-7b322f03969f could not be found. [ 2313.465749] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2313.465749] env[63273]: INFO nova.compute.manager [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2313.466015] env[63273]: DEBUG oslo.service.loopingcall [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2313.466196] env[63273]: DEBUG nova.compute.manager [-] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2313.466293] env[63273]: DEBUG nova.network.neutron [-] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2313.492425] env[63273]: DEBUG nova.network.neutron [-] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2313.500780] env[63273]: INFO nova.compute.manager [-] [instance: fd40e450-e51f-41a9-ba86-7b322f03969f] Took 0.03 seconds to deallocate network for instance. [ 2313.589774] env[63273]: DEBUG oslo_concurrency.lockutils [None req-2b1b1892-e3c9-464b-b188-3a0c499ed1a5 tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "fd40e450-e51f-41a9-ba86-7b322f03969f" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.169s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2314.860156] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "519b4a29-29a0-41b0-8d78-20a403b0cba8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2314.860559] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "519b4a29-29a0-41b0-8d78-20a403b0cba8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2314.872843] env[63273]: DEBUG nova.compute.manager [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Starting instance... {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2424}} [ 2314.935138] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2314.935405] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2314.936891] env[63273]: INFO nova.compute.claims [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2315.035132] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f6003aa-e967-4840-9304-495c3d019099 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2315.043504] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d69a1fd-3733-4c07-a0ef-e6d42ec08e9c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2315.074295] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-791868c1-6ced-4d1b-9cd9-6eb39134dd9f {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2315.081776] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d2f5b61-8723-4b9f-aac9-bc6fd076ffb3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2315.094877] env[63273]: DEBUG nova.compute.provider_tree [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2315.123905] env[63273]: DEBUG nova.scheduler.client.report [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2315.138158] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.203s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2315.138657] env[63273]: DEBUG nova.compute.manager [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Start building networks asynchronously for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2821}} [ 2315.170050] env[63273]: DEBUG nova.compute.utils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Using /dev/sd instead of None {{(pid=63273) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2315.171377] env[63273]: DEBUG nova.compute.manager [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Allocating IP information in the background. {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1973}} [ 2315.171578] env[63273]: DEBUG nova.network.neutron [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] allocate_for_instance() {{(pid=63273) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 2315.180373] env[63273]: DEBUG nova.compute.manager [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Start building block device mappings for instance. {{(pid=63273) _build_resources /opt/stack/nova/nova/compute/manager.py:2856}} [ 2315.245240] env[63273]: DEBUG nova.policy [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e1ab27df058544c496bd2a078b84e6e8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3387519998ea4ed8a586caeaafed2446', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63273) authorize /opt/stack/nova/nova/policy.py:203}} [ 2315.252373] env[63273]: DEBUG nova.compute.manager [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Start spawning the instance on the hypervisor. {{(pid=63273) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2630}} [ 2315.277626] env[63273]: DEBUG nova.virt.hardware [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-12-11T09:03:01Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-12-11T09:02:45Z,direct_url=,disk_format='vmdk',id=443b5631-8941-44d6-8565-68279d3c04e8,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='3ebfa0b99d284c0081f368c7c447f2dd',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-12-11T09:02:46Z,virtual_size=,visibility=), allow threads: False {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 2315.277952] env[63273]: DEBUG nova.virt.hardware [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Flavor limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 2315.278169] env[63273]: DEBUG nova.virt.hardware [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Image limits 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2315.278430] env[63273]: DEBUG nova.virt.hardware [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Flavor pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 2315.278665] env[63273]: DEBUG nova.virt.hardware [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Image pref 0:0:0 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2315.278869] env[63273]: DEBUG nova.virt.hardware [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63273) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 2315.279095] env[63273]: DEBUG nova.virt.hardware [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 2315.279270] env[63273]: DEBUG nova.virt.hardware [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 2315.279465] env[63273]: DEBUG nova.virt.hardware [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Got 1 possible topologies {{(pid=63273) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 2315.279606] env[63273]: DEBUG nova.virt.hardware [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 2315.279780] env[63273]: DEBUG nova.virt.hardware [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63273) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 2315.280656] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14ecf3b0-a169-4ec9-bdbf-36846b9359c3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2315.289166] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b690e5d-16c1-4bca-829b-f3d33790153d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2315.635166] env[63273]: DEBUG nova.network.neutron [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Successfully created port: 922366c7-5c56-4c68-81f6-4f36543128d9 {{(pid=63273) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2316.284598] env[63273]: DEBUG nova.compute.manager [req-6921c991-d499-48bc-8646-0ee45e49360d req-9af9395c-59b0-4744-8968-f05116115537 service nova] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Received event network-vif-plugged-922366c7-5c56-4c68-81f6-4f36543128d9 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 2316.284881] env[63273]: DEBUG oslo_concurrency.lockutils [req-6921c991-d499-48bc-8646-0ee45e49360d req-9af9395c-59b0-4744-8968-f05116115537 service nova] Acquiring lock "519b4a29-29a0-41b0-8d78-20a403b0cba8-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2316.285066] env[63273]: DEBUG oslo_concurrency.lockutils [req-6921c991-d499-48bc-8646-0ee45e49360d req-9af9395c-59b0-4744-8968-f05116115537 service nova] Lock "519b4a29-29a0-41b0-8d78-20a403b0cba8-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2316.285242] env[63273]: DEBUG oslo_concurrency.lockutils [req-6921c991-d499-48bc-8646-0ee45e49360d req-9af9395c-59b0-4744-8968-f05116115537 service nova] Lock "519b4a29-29a0-41b0-8d78-20a403b0cba8-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2316.285412] env[63273]: DEBUG nova.compute.manager [req-6921c991-d499-48bc-8646-0ee45e49360d req-9af9395c-59b0-4744-8968-f05116115537 service nova] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] No waiting events found dispatching network-vif-plugged-922366c7-5c56-4c68-81f6-4f36543128d9 {{(pid=63273) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2316.285579] env[63273]: WARNING nova.compute.manager [req-6921c991-d499-48bc-8646-0ee45e49360d req-9af9395c-59b0-4744-8968-f05116115537 service nova] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Received unexpected event network-vif-plugged-922366c7-5c56-4c68-81f6-4f36543128d9 for instance with vm_state building and task_state spawning. [ 2316.365611] env[63273]: DEBUG nova.network.neutron [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Successfully updated port: 922366c7-5c56-4c68-81f6-4f36543128d9 {{(pid=63273) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2316.378155] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "refresh_cache-519b4a29-29a0-41b0-8d78-20a403b0cba8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2316.378155] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired lock "refresh_cache-519b4a29-29a0-41b0-8d78-20a403b0cba8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2316.378332] env[63273]: DEBUG nova.network.neutron [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Building network info cache for instance {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2316.422091] env[63273]: DEBUG nova.network.neutron [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Instance cache missing network info. {{(pid=63273) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2316.632838] env[63273]: DEBUG nova.network.neutron [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Updating instance_info_cache with network_info: [{"id": "922366c7-5c56-4c68-81f6-4f36543128d9", "address": "fa:16:3e:a1:92:a7", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.12", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap922366c7-5c", "ovs_interfaceid": "922366c7-5c56-4c68-81f6-4f36543128d9", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2316.644568] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Releasing lock "refresh_cache-519b4a29-29a0-41b0-8d78-20a403b0cba8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2316.644880] env[63273]: DEBUG nova.compute.manager [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Instance network_info: |[{"id": "922366c7-5c56-4c68-81f6-4f36543128d9", "address": "fa:16:3e:a1:92:a7", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.12", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap922366c7-5c", "ovs_interfaceid": "922366c7-5c56-4c68-81f6-4f36543128d9", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63273) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2316.645546] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:a1:92:a7', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'dad4f433-bb0b-45c7-8040-972ef2277f75', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '922366c7-5c56-4c68-81f6-4f36543128d9', 'vif_model': 'vmxnet3'}] {{(pid=63273) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2316.654024] env[63273]: DEBUG oslo.service.loopingcall [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2316.654024] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Creating VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2316.654024] env[63273]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-c6d295bd-6746-4c06-90a8-5730afff2803 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2316.675036] env[63273]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2316.675036] env[63273]: value = "task-5072207" [ 2316.675036] env[63273]: _type = "Task" [ 2316.675036] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2316.684040] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072207, 'name': CreateVM_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2317.187626] env[63273]: DEBUG oslo_vmware.api [-] Task: {'id': task-5072207, 'name': CreateVM_Task, 'duration_secs': 0.302528} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2317.187939] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Created VM on the ESX host {{(pid=63273) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2317.188422] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2317.188585] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2317.188952] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2317.189241] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-3a30b629-0fd3-4cce-b9be-ffd06d348f6c {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2317.194761] env[63273]: DEBUG oslo_vmware.api [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Waiting for the task: (returnval){ [ 2317.194761] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]523e6e9d-8710-95c5-fb3b-98464c507d3d" [ 2317.194761] env[63273]: _type = "Task" [ 2317.194761] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2317.203138] env[63273]: DEBUG oslo_vmware.api [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]523e6e9d-8710-95c5-fb3b-98464c507d3d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2317.705180] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2317.705504] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Processing image 443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2317.705660] env[63273]: DEBUG oslo_concurrency.lockutils [None req-570e987f-7e46-4119-bc9e-223f42a423de tempest-ServersTestJSON-284632755 tempest-ServersTestJSON-284632755-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2318.310639] env[63273]: DEBUG nova.compute.manager [req-829003c2-2b76-4719-8789-16f4437dd011 req-1f173554-d845-480d-a149-d8b0558078cb service nova] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Received event network-changed-922366c7-5c56-4c68-81f6-4f36543128d9 {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11210}} [ 2318.310801] env[63273]: DEBUG nova.compute.manager [req-829003c2-2b76-4719-8789-16f4437dd011 req-1f173554-d845-480d-a149-d8b0558078cb service nova] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Refreshing instance network info cache due to event network-changed-922366c7-5c56-4c68-81f6-4f36543128d9. {{(pid=63273) external_instance_event /opt/stack/nova/nova/compute/manager.py:11215}} [ 2318.311027] env[63273]: DEBUG oslo_concurrency.lockutils [req-829003c2-2b76-4719-8789-16f4437dd011 req-1f173554-d845-480d-a149-d8b0558078cb service nova] Acquiring lock "refresh_cache-519b4a29-29a0-41b0-8d78-20a403b0cba8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2318.311178] env[63273]: DEBUG oslo_concurrency.lockutils [req-829003c2-2b76-4719-8789-16f4437dd011 req-1f173554-d845-480d-a149-d8b0558078cb service nova] Acquired lock "refresh_cache-519b4a29-29a0-41b0-8d78-20a403b0cba8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2318.311380] env[63273]: DEBUG nova.network.neutron [req-829003c2-2b76-4719-8789-16f4437dd011 req-1f173554-d845-480d-a149-d8b0558078cb service nova] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Refreshing network info cache for port 922366c7-5c56-4c68-81f6-4f36543128d9 {{(pid=63273) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2318.566268] env[63273]: DEBUG nova.network.neutron [req-829003c2-2b76-4719-8789-16f4437dd011 req-1f173554-d845-480d-a149-d8b0558078cb service nova] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Updated VIF entry in instance network info cache for port 922366c7-5c56-4c68-81f6-4f36543128d9. {{(pid=63273) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 2318.566728] env[63273]: DEBUG nova.network.neutron [req-829003c2-2b76-4719-8789-16f4437dd011 req-1f173554-d845-480d-a149-d8b0558078cb service nova] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Updating instance_info_cache with network_info: [{"id": "922366c7-5c56-4c68-81f6-4f36543128d9", "address": "fa:16:3e:a1:92:a7", "network": {"id": "f380e7ee-ec92-40b8-8f7b-94b330cdba47", "bridge": "br-int", "label": "tempest-ServersTestJSON-720649318-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.12", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3387519998ea4ed8a586caeaafed2446", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dad4f433-bb0b-45c7-8040-972ef2277f75", "external-id": "nsx-vlan-transportzone-451", "segmentation_id": 451, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap922366c7-5c", "ovs_interfaceid": "922366c7-5c56-4c68-81f6-4f36543128d9", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2318.576160] env[63273]: DEBUG oslo_concurrency.lockutils [req-829003c2-2b76-4719-8789-16f4437dd011 req-1f173554-d845-480d-a149-d8b0558078cb service nova] Releasing lock "refresh_cache-519b4a29-29a0-41b0-8d78-20a403b0cba8" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2320.892483] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2322.899734] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2324.891302] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2324.903477] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2324.903699] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2324.903870] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2324.904033] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2324.905219] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73cdd005-5e37-4623-8830-5627e3eeec60 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.914122] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f42a098-901e-4187-b6d6-353e52efbb31 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.928032] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9dd01518-1f0c-41af-a9f7-42d474bf7c54 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.934667] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e663c890-c875-40b5-8520-1aac4f356976 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.964294] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180548MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2324.964434] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2324.964642] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2325.018374] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 415fad0a-0af4-45b0-a958-ff900fbcb05b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2325.018531] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8afeca90-bf18-45ad-8b7e-d1a22632e0b0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2325.018684] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 519b4a29-29a0-41b0-8d78-20a403b0cba8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2325.018885] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 3 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2325.019097] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=896MB phys_disk=100GB used_disk=3GB total_vcpus=48 used_vcpus=3 pci_stats=[] stats={'failed_builds': '90', 'num_instances': '3', 'num_vm_building': '3', 'num_task_deleting': '2', 'num_os_type_None': '3', 'num_proj_ff660b4621c14dc48d0e1fad4c290dcc': '1', 'io_workload': '3', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'num_task_spawning': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2325.067233] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f22add58-b765-4661-b867-d218ad464f07 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2325.074625] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f03adccd-8dc9-45eb-b5b2-be0123247550 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2325.103911] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2e221c7-c88b-4bd5-a105-e45b7d63cf03 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2325.111484] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8981351-c251-4c90-95c1-ac1687cec405 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2325.124242] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2325.134372] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2325.149363] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2325.149548] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.185s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2326.150176] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2326.150565] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2326.892125] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2326.892374] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2326.892525] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 2327.894178] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2327.894178] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 2327.894178] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 2327.905770] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2327.905932] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2327.906083] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2327.906215] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 2328.901051] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2330.891992] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2333.893072] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2333.893385] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances with incomplete migration {{(pid=63273) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11345}} [ 2337.895636] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2340.894476] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2340.894842] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Cleaning up deleted instances {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11307}} [ 2340.904741] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] There are 0 instances to clean {{(pid=63273) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11316}} [ 2361.313412] env[63273]: WARNING oslo_vmware.rw_handles [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2361.313412] env[63273]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2361.313412] env[63273]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2361.313412] env[63273]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2361.313412] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2361.313412] env[63273]: ERROR oslo_vmware.rw_handles response.begin() [ 2361.313412] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2361.313412] env[63273]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2361.313412] env[63273]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2361.313412] env[63273]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2361.313412] env[63273]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2361.313412] env[63273]: ERROR oslo_vmware.rw_handles [ 2361.314168] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Downloaded image file data 443b5631-8941-44d6-8565-68279d3c04e8 to vmware_temp/10339c8c-d99a-49f8-9c0d-60e00698dcb4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2361.315840] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Caching image {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2361.316102] env[63273]: DEBUG nova.virt.vmwareapi.vm_util [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Copying Virtual Disk [datastore1] vmware_temp/10339c8c-d99a-49f8-9c0d-60e00698dcb4/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk to [datastore1] vmware_temp/10339c8c-d99a-49f8-9c0d-60e00698dcb4/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk {{(pid=63273) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2361.316400] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-9c9bc8ab-8a6b-4f20-960e-c785b8c64590 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2361.325201] env[63273]: DEBUG oslo_vmware.api [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for the task: (returnval){ [ 2361.325201] env[63273]: value = "task-5072208" [ 2361.325201] env[63273]: _type = "Task" [ 2361.325201] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2361.333541] env[63273]: DEBUG oslo_vmware.api [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Task: {'id': task-5072208, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2361.836067] env[63273]: DEBUG oslo_vmware.exceptions [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Fault InvalidArgument not matched. {{(pid=63273) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2361.836279] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Releasing lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2361.836858] env[63273]: ERROR nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2361.836858] env[63273]: Faults: ['InvalidArgument'] [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Traceback (most recent call last): [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/compute/manager.py", line 2886, in _build_resources [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] yield resources [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] self.driver.spawn(context, instance, image_meta, [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] self._fetch_image_if_missing(context, vi) [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] image_cache(vi, tmp_image_ds_loc) [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] vm_util.copy_virtual_disk( [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] session._wait_for_task(vmdk_copy_task) [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] return self.wait_for_task(task_ref) [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] return evt.wait() [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] result = hub.switch() [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] return self.greenlet.switch() [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] self.f(*self.args, **self.kw) [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] raise exceptions.translate_fault(task_info.error) [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Faults: ['InvalidArgument'] [ 2361.836858] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] [ 2361.838011] env[63273]: INFO nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Terminating instance [ 2361.838790] env[63273]: DEBUG oslo_concurrency.lockutils [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Acquired lock "[datastore1] devstack-image-cache_base/443b5631-8941-44d6-8565-68279d3c04e8/443b5631-8941-44d6-8565-68279d3c04e8.vmdk" {{(pid=63273) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2361.838990] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2361.839257] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-3adf57c8-c2a3-45b5-97d2-f6b53236c5d3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2361.841437] env[63273]: DEBUG nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2361.841627] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2361.842353] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b081b777-02c4-4718-9b2e-4481c6e90cb1 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2361.849642] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Unregistering the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2361.849884] env[63273]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-32265308-35e8-429f-aded-efd6363cf161 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2361.852083] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2361.852258] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=63273) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2361.853241] env[63273]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-7121f175-f882-4615-8623-c9b31d2f2fbe {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2361.857862] env[63273]: DEBUG oslo_vmware.api [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Waiting for the task: (returnval){ [ 2361.857862] env[63273]: value = "session[52e20218-13c4-e4f3-2632-ee5f13be3227]524742bc-e2cc-ca1c-dd96-59ffa65e0d1a" [ 2361.857862] env[63273]: _type = "Task" [ 2361.857862] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2361.865568] env[63273]: DEBUG oslo_vmware.api [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Task: {'id': session[52e20218-13c4-e4f3-2632-ee5f13be3227]524742bc-e2cc-ca1c-dd96-59ffa65e0d1a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2361.924107] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Unregistered the VM {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2361.924393] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Deleting contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2361.924536] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Deleting the datastore file [datastore1] 415fad0a-0af4-45b0-a958-ff900fbcb05b {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2361.924809] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-e5dcd7e6-505e-4bd8-b714-f435184a0698 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2361.931435] env[63273]: DEBUG oslo_vmware.api [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for the task: (returnval){ [ 2361.931435] env[63273]: value = "task-5072210" [ 2361.931435] env[63273]: _type = "Task" [ 2361.931435] env[63273]: } to complete. {{(pid=63273) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2361.942101] env[63273]: DEBUG oslo_vmware.api [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Task: {'id': task-5072210, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2362.368320] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Preparing fetch location {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2362.368738] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating directory with path [datastore1] vmware_temp/1deb8844-5eaa-4519-93e3-182963c59d19/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2362.368833] env[63273]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-4faedcb1-b807-4274-aeeb-4f6552f0bc07 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2362.381066] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Created directory with path [datastore1] vmware_temp/1deb8844-5eaa-4519-93e3-182963c59d19/443b5631-8941-44d6-8565-68279d3c04e8 {{(pid=63273) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2362.381265] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Fetch image to [datastore1] vmware_temp/1deb8844-5eaa-4519-93e3-182963c59d19/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk {{(pid=63273) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2362.381467] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to [datastore1] vmware_temp/1deb8844-5eaa-4519-93e3-182963c59d19/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk on the data store datastore1 {{(pid=63273) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2362.382227] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3972f11-f174-4f27-a65c-0b82af744ccc {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2362.389278] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b70ad05-84ce-4d4a-b847-db89b2004504 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2362.400054] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c061b6d-5e78-4560-8fb4-392c5d81cc23 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2362.431624] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a247ae3b-9bfc-4d77-bc5b-e3a0a21eb5c3 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2362.441695] env[63273]: DEBUG oslo_vmware.api [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Task: {'id': task-5072210, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.086306} completed successfully. {{(pid=63273) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2362.443181] env[63273]: DEBUG nova.virt.vmwareapi.ds_util [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Deleted the datastore file {{(pid=63273) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2362.443379] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Deleted contents of the VM from datastore datastore1 {{(pid=63273) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2362.443552] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2362.443724] env[63273]: INFO nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2362.445566] env[63273]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-cb5fd7f3-fbbb-4d04-8263-f8ad8a748247 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2362.447503] env[63273]: DEBUG nova.compute.claims [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Aborting claim: {{(pid=63273) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2362.447681] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2362.447894] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2362.471927] env[63273]: DEBUG nova.virt.vmwareapi.images [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Downloading image file data 443b5631-8941-44d6-8565-68279d3c04e8 to the data store datastore1 {{(pid=63273) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2362.551129] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b45ed0ff-9cf2-4d57-8cf1-3483920868fb {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2362.558881] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee349759-97ee-4249-878d-03f84adc6820 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2362.591638] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18846947-5cc8-4c38-95e4-6c51a11e17ee {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2362.601709] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b11f8e1-2c20-4a38-ac7c-1abb621cf98b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2362.618018] env[63273]: DEBUG nova.compute.provider_tree [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2362.627199] env[63273]: DEBUG nova.scheduler.client.report [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2362.634758] env[63273]: DEBUG oslo_vmware.rw_handles [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1deb8844-5eaa-4519-93e3-182963c59d19/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2362.691252] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.243s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2362.691768] env[63273]: ERROR nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2362.691768] env[63273]: Faults: ['InvalidArgument'] [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Traceback (most recent call last): [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/compute/manager.py", line 2633, in _build_and_run_instance [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] self.driver.spawn(context, instance, image_meta, [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] self._fetch_image_if_missing(context, vi) [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] image_cache(vi, tmp_image_ds_loc) [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] vm_util.copy_virtual_disk( [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] session._wait_for_task(vmdk_copy_task) [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] return self.wait_for_task(task_ref) [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] return evt.wait() [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] result = hub.switch() [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] return self.greenlet.switch() [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] self.f(*self.args, **self.kw) [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] raise exceptions.translate_fault(task_info.error) [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Faults: ['InvalidArgument'] [ 2362.691768] env[63273]: ERROR nova.compute.manager [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] [ 2362.692586] env[63273]: DEBUG nova.compute.utils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] VimFaultException {{(pid=63273) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2362.695632] env[63273]: DEBUG nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Build of instance 415fad0a-0af4-45b0-a958-ff900fbcb05b was re-scheduled: A specified parameter was not correct: fileType [ 2362.695632] env[63273]: Faults: ['InvalidArgument'] {{(pid=63273) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2472}} [ 2362.696095] env[63273]: DEBUG nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Unplugging VIFs for instance {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2998}} [ 2362.696283] env[63273]: DEBUG nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63273) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3021}} [ 2362.696455] env[63273]: DEBUG nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2362.696622] env[63273]: DEBUG nova.network.neutron [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2362.699121] env[63273]: DEBUG oslo_vmware.rw_handles [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Completed reading data from the image iterator. {{(pid=63273) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2362.699307] env[63273]: DEBUG oslo_vmware.rw_handles [None req-25ca1afd-0180-49ca-8902-80acd46ea96b tempest-DeleteServersTestJSON-1893240210 tempest-DeleteServersTestJSON-1893240210-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1deb8844-5eaa-4519-93e3-182963c59d19/443b5631-8941-44d6-8565-68279d3c04e8/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=63273) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2363.085134] env[63273]: DEBUG nova.network.neutron [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2363.099045] env[63273]: INFO nova.compute.manager [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Took 0.40 seconds to deallocate network for instance. [ 2363.206782] env[63273]: INFO nova.scheduler.client.report [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Deleted allocations for instance 415fad0a-0af4-45b0-a958-ff900fbcb05b [ 2363.229215] env[63273]: DEBUG oslo_concurrency.lockutils [None req-f9390a04-3fc7-4642-80ea-e671d5a4076a tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "415fad0a-0af4-45b0-a958-ff900fbcb05b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 334.091s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2363.229215] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "415fad0a-0af4-45b0-a958-ff900fbcb05b" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 138.469s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2363.229215] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Acquiring lock "415fad0a-0af4-45b0-a958-ff900fbcb05b-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2363.229215] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "415fad0a-0af4-45b0-a958-ff900fbcb05b-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2363.229446] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "415fad0a-0af4-45b0-a958-ff900fbcb05b-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2363.231342] env[63273]: INFO nova.compute.manager [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Terminating instance [ 2363.233070] env[63273]: DEBUG nova.compute.manager [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Start destroying the instance on the hypervisor. {{(pid=63273) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3142}} [ 2363.233275] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Destroying instance {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2363.233748] env[63273]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7971f44b-388b-4b46-b69f-0dfe6c23c87d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2363.243112] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eabecb3a-2eeb-4145-b572-95c06971d979 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2363.273854] env[63273]: WARNING nova.virt.vmwareapi.vmops [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 415fad0a-0af4-45b0-a958-ff900fbcb05b could not be found. [ 2363.274074] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Instance destroyed {{(pid=63273) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2363.274223] env[63273]: INFO nova.compute.manager [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2363.274473] env[63273]: DEBUG oslo.service.loopingcall [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63273) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2363.274706] env[63273]: DEBUG nova.compute.manager [-] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Deallocating network for instance {{(pid=63273) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2281}} [ 2363.274803] env[63273]: DEBUG nova.network.neutron [-] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] deallocate_for_instance() {{(pid=63273) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2363.302411] env[63273]: DEBUG nova.network.neutron [-] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Updating instance_info_cache with network_info: [] {{(pid=63273) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2363.310909] env[63273]: INFO nova.compute.manager [-] [instance: 415fad0a-0af4-45b0-a958-ff900fbcb05b] Took 0.04 seconds to deallocate network for instance. [ 2363.394719] env[63273]: DEBUG oslo_concurrency.lockutils [None req-0a8bb94f-f87f-4d25-9e3c-219129b3ea38 tempest-SecurityGroupsTestJSON-1231985216 tempest-SecurityGroupsTestJSON-1231985216-project-member] Lock "415fad0a-0af4-45b0-a958-ff900fbcb05b" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.165s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2365.021754] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._cleanup_running_deleted_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2365.022171] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Getting list of instances from cluster (obj){ [ 2365.022171] env[63273]: value = "domain-c8" [ 2365.022171] env[63273]: _type = "ClusterComputeResource" [ 2365.022171] env[63273]: } {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 2365.023149] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58055555-26ad-4d5f-ae54-effa43027068 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2365.034354] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Got total of 2 instances {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 2381.201639] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._sync_power_states {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2381.216851] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Getting list of instances from cluster (obj){ [ 2381.216851] env[63273]: value = "domain-c8" [ 2381.216851] env[63273]: _type = "ClusterComputeResource" [ 2381.216851] env[63273]: } {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 2381.218216] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9633a40f-92ca-4a9a-9709-877869940541 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2381.230023] env[63273]: DEBUG nova.virt.vmwareapi.vmops [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Got total of 2 instances {{(pid=63273) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 2381.230219] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 8afeca90-bf18-45ad-8b7e-d1a22632e0b0 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 2381.230415] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Triggering sync for uuid 519b4a29-29a0-41b0-8d78-20a403b0cba8 {{(pid=63273) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10417}} [ 2381.230752] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "8afeca90-bf18-45ad-8b7e-d1a22632e0b0" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2381.231016] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "519b4a29-29a0-41b0-8d78-20a403b0cba8" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2382.921797] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2385.892468] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2386.892241] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2386.892409] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2386.892589] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager.update_available_resource {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2386.904666] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2386.904883] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2386.905077] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2386.905240] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63273) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2386.906317] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f1451f4-d390-42c9-a165-7aeb83ed9909 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2386.915218] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e439753-5b6d-406f-ae5d-fa80b79abab0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2386.929447] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac7cc259-249a-4f3b-a6f4-af6f5af0f06d {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2386.936358] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74a6ed92-69ab-4c7f-85ce-80712ed3dab5 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2386.967057] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180546MB free_disk=96GB free_vcpus=48 pci_devices=None {{(pid=63273) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2386.967205] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2386.967398] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2387.127629] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 8afeca90-bf18-45ad-8b7e-d1a22632e0b0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2387.127797] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Instance 519b4a29-29a0-41b0-8d78-20a403b0cba8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=63273) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1710}} [ 2387.127995] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Total usable vcpus: 48, total allocated vcpus: 2 {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2387.128166] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=768MB phys_disk=100GB used_disk=2GB total_vcpus=48 used_vcpus=2 pci_stats=[] stats={'failed_builds': '91', 'num_instances': '2', 'num_vm_building': '2', 'num_task_deleting': '1', 'num_os_type_None': '2', 'num_proj_495ae26325184812a744fc95da67038b': '1', 'io_workload': '2', 'num_task_spawning': '1', 'num_proj_3387519998ea4ed8a586caeaafed2446': '1'} {{(pid=63273) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2387.144874] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Refreshing inventories for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 2387.158012] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Updating ProviderTree inventory for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 2387.158227] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Updating inventory in ProviderTree for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2387.168836] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Refreshing aggregate associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, aggregates: None {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 2387.187561] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Refreshing trait associations for resource provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NODE,COMPUTE_IMAGE_TYPE_ISO {{(pid=63273) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 2387.225682] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c64b8822-dfb9-4789-a911-47058b191191 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2387.233502] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-828cf9d6-9424-431b-bdd2-af8110a09495 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2387.263047] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-414d1de3-fed0-4d49-ab0d-8cf3545c937b {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2387.270560] env[63273]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4556ad68-847f-4ac7-871a-e5c631e306c0 {{(pid=63273) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2387.283748] env[63273]: DEBUG nova.compute.provider_tree [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed in ProviderTree for provider: 34a0ca39-8974-44d6-ab34-3ab6cf8432cd {{(pid=63273) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2387.294108] env[63273]: DEBUG nova.scheduler.client.report [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Inventory has not changed for provider 34a0ca39-8974-44d6-ab34-3ab6cf8432cd based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 200, 'reserved': 0, 'min_unit': 1, 'max_unit': 96, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63273) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 2387.307511] env[63273]: DEBUG nova.compute.resource_tracker [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63273) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2387.307718] env[63273]: DEBUG oslo_concurrency.lockutils [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.340s {{(pid=63273) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2388.307080] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2388.307447] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63273) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10626}} [ 2389.892474] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2389.892875] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Starting heal instance info cache {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10007}} [ 2389.892875] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Rebuilding the list of instances to heal {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10011}} [ 2389.906011] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 8afeca90-bf18-45ad-8b7e-d1a22632e0b0] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2389.906242] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] [instance: 519b4a29-29a0-41b0-8d78-20a403b0cba8] Skipping network cache update for instance because it is Building. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10020}} [ 2389.906321] env[63273]: DEBUG nova.compute.manager [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Didn't find any instances for network info cache update. {{(pid=63273) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10093}} [ 2391.891702] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2398.887696] env[63273]: DEBUG oslo_service.periodic_task [None req-ebd690c6-2c55-480a-8baf-5dbea250b69f None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63273) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}}